2023-11-16T13:17:12.142 INFO:root:teuthology version: 0.0.1.dev203+g6899cd2 2023-11-16T13:17:12.142 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi/7460760 --description orch:cephadm/mgr-nfs-upgrade/{0-centos_8.stream_container_tools 1-bootstrap/16.2.5 1-start 2-nfs 3-upgrade-with-workload 4-final} --name gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi --verbose --owner scheduled_gabrioux@teuthology -- /home/teuthworker/archive/gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi/7460760/orig.config.yaml 2023-11-16T13:17:12.204 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-11-16T13:17:12.304 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi/7460760 branch: wip-63555-quincy description: orch:cephadm/mgr-nfs-upgrade/{0-centos_8.stream_container_tools 1-bootstrap/16.2.5 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7460760' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: wip-63555-quincy ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-63555-quincy sha1: ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 3535 sha1: ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-63555-quincy suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_ae6ce92c0812ba06c9c623ce72df4f3fc59080b4/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 targets: smithi151.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOOEwtsar9VCyDQay7IPHqvHGH7/5fuPhSSq4FZmLsqmt1GnPFD6ECu0G16IhUHTSZNJ+9KbkoHhryOOSTUdjXA= smithi167.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGr1oPEINZRdFr3f9EY1PZNMUW1ni/iB7dWdEDYh9HGQQmKKAde/5gS6jvaLsVu0Z5kMFfiAMIPgmWunjne15I8= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - cephadm: cephadm_branch: v16.2.5 cephadm_git_url: https://github.com/ceph/ceph image: docker.io/ceph/ceph:v16.2.5 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - ceph config get mgr mgr/cephadm/migration_current | grep 5 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6899cd26fceddb2fec83dc1a1349394b28c8998e timestamp: 2023-11-16_12:07:54 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.1224587 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2023-11-16T13:17:12.304 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_ae6ce92c0812ba06c9c623ce72df4f3fc59080b4/qa; will attempt to use it 2023-11-16T13:17:12.306 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_ae6ce92c0812ba06c9c623ce72df4f3fc59080b4/qa/tasks 2023-11-16T13:17:12.306 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-11-16T13:17:12.308 INFO:teuthology.task.internal:Checking packages... 2023-11-16T13:17:12.337 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'ae6ce92c0812ba06c9c623ce72df4f3fc59080b4' 2023-11-16T13:17:12.337 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-11-16T13:17:12.337 INFO:teuthology.packaging:ref: None 2023-11-16T13:17:12.337 INFO:teuthology.packaging:tag: None 2023-11-16T13:17:12.337 INFO:teuthology.packaging:branch: wip-63555-quincy 2023-11-16T13:17:12.337 INFO:teuthology.packaging:sha1: ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 2023-11-16T13:17:12.337 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=wip-63555-quincy 2023-11-16T13:17:12.479 INFO:teuthology.task.internal:Found packages for ceph version 17.2.7-68.gae6ce92c 2023-11-16T13:17:12.479 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-11-16T13:17:12.486 INFO:teuthology.task.internal:no buildpackages task found 2023-11-16T13:17:12.486 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-11-16T13:17:12.525 INFO:teuthology.task.internal:Saving configuration 2023-11-16T13:17:12.540 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-11-16T13:17:12.549 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-11-16T13:17:12.575 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi151.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi/7460760', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-11-16 13:11:54.539134', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOOEwtsar9VCyDQay7IPHqvHGH7/5fuPhSSq4FZmLsqmt1GnPFD6ECu0G16IhUHTSZNJ+9KbkoHhryOOSTUdjXA='} 2023-11-16T13:17:12.599 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi167.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi/7460760', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-11-16 13:11:54.540568', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGr1oPEINZRdFr3f9EY1PZNMUW1ni/iB7dWdEDYh9HGQQmKKAde/5gS6jvaLsVu0Z5kMFfiAMIPgmWunjne15I8='} 2023-11-16T13:17:12.602 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-11-16T13:17:12.610 INFO:teuthology.task.internal:roles: ubuntu@smithi151.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2023-11-16T13:17:12.610 INFO:teuthology.task.internal:roles: ubuntu@smithi167.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2023-11-16T13:17:12.610 INFO:teuthology.run_tasks:Running task console_log... 2023-11-16T13:17:12.704 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fe69d8caf70>, signals=[15]) 2023-11-16T13:17:12.705 INFO:teuthology.run_tasks:Running task internal.connect... 2023-11-16T13:17:12.715 INFO:teuthology.task.internal:Opening connections... 2023-11-16T13:17:12.715 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi151.front.sepia.ceph.com 2023-11-16T13:17:12.717 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-16T13:17:12.786 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi167.front.sepia.ceph.com 2023-11-16T13:17:12.787 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-16T13:17:12.859 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-11-16T13:17:12.865 DEBUG:teuthology.orchestra.run.smithi151:> uname -m 2023-11-16T13:17:12.887 INFO:teuthology.orchestra.run.smithi151.stdout:x86_64 2023-11-16T13:17:12.888 DEBUG:teuthology.orchestra.run.smithi151:> cat /etc/os-release 2023-11-16T13:17:12.948 INFO:teuthology.orchestra.run.smithi151.stdout:NAME="CentOS Stream" 2023-11-16T13:17:12.948 INFO:teuthology.orchestra.run.smithi151.stdout:VERSION="8" 2023-11-16T13:17:12.949 INFO:teuthology.orchestra.run.smithi151.stdout:ID="centos" 2023-11-16T13:17:12.949 INFO:teuthology.orchestra.run.smithi151.stdout:ID_LIKE="rhel fedora" 2023-11-16T13:17:12.949 INFO:teuthology.orchestra.run.smithi151.stdout:VERSION_ID="8" 2023-11-16T13:17:12.949 INFO:teuthology.orchestra.run.smithi151.stdout:PLATFORM_ID="platform:el8" 2023-11-16T13:17:12.949 INFO:teuthology.orchestra.run.smithi151.stdout:PRETTY_NAME="CentOS Stream 8" 2023-11-16T13:17:12.949 INFO:teuthology.orchestra.run.smithi151.stdout:ANSI_COLOR="0;31" 2023-11-16T13:17:12.949 INFO:teuthology.orchestra.run.smithi151.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-11-16T13:17:12.949 INFO:teuthology.orchestra.run.smithi151.stdout:HOME_URL="https://centos.org/" 2023-11-16T13:17:12.949 INFO:teuthology.orchestra.run.smithi151.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-11-16T13:17:12.949 INFO:teuthology.orchestra.run.smithi151.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-11-16T13:17:12.949 INFO:teuthology.orchestra.run.smithi151.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-11-16T13:17:12.950 INFO:teuthology.lock.ops:Updating smithi151.front.sepia.ceph.com on lock server 2023-11-16T13:17:12.971 DEBUG:teuthology.orchestra.run.smithi167:> uname -m 2023-11-16T13:17:12.993 INFO:teuthology.orchestra.run.smithi167.stdout:x86_64 2023-11-16T13:17:12.994 DEBUG:teuthology.orchestra.run.smithi167:> cat /etc/os-release 2023-11-16T13:17:13.054 INFO:teuthology.orchestra.run.smithi167.stdout:NAME="CentOS Stream" 2023-11-16T13:17:13.054 INFO:teuthology.orchestra.run.smithi167.stdout:VERSION="8" 2023-11-16T13:17:13.054 INFO:teuthology.orchestra.run.smithi167.stdout:ID="centos" 2023-11-16T13:17:13.055 INFO:teuthology.orchestra.run.smithi167.stdout:ID_LIKE="rhel fedora" 2023-11-16T13:17:13.055 INFO:teuthology.orchestra.run.smithi167.stdout:VERSION_ID="8" 2023-11-16T13:17:13.055 INFO:teuthology.orchestra.run.smithi167.stdout:PLATFORM_ID="platform:el8" 2023-11-16T13:17:13.055 INFO:teuthology.orchestra.run.smithi167.stdout:PRETTY_NAME="CentOS Stream 8" 2023-11-16T13:17:13.055 INFO:teuthology.orchestra.run.smithi167.stdout:ANSI_COLOR="0;31" 2023-11-16T13:17:13.055 INFO:teuthology.orchestra.run.smithi167.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-11-16T13:17:13.055 INFO:teuthology.orchestra.run.smithi167.stdout:HOME_URL="https://centos.org/" 2023-11-16T13:17:13.055 INFO:teuthology.orchestra.run.smithi167.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-11-16T13:17:13.056 INFO:teuthology.orchestra.run.smithi167.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-11-16T13:17:13.056 INFO:teuthology.orchestra.run.smithi167.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-11-16T13:17:13.056 INFO:teuthology.lock.ops:Updating smithi167.front.sepia.ceph.com on lock server 2023-11-16T13:17:13.079 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-11-16T13:17:13.088 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-11-16T13:17:13.116 INFO:teuthology.task.internal:Checking for old test directory... 2023-11-16T13:17:13.116 DEBUG:teuthology.orchestra.run.smithi151:> test '!' -e /home/ubuntu/cephtest 2023-11-16T13:17:13.118 DEBUG:teuthology.orchestra.run.smithi167:> test '!' -e /home/ubuntu/cephtest 2023-11-16T13:17:13.138 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-11-16T13:17:13.146 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-11-16T13:17:13.146 DEBUG:teuthology.orchestra.run.smithi151:> test -z $(ls -A /var/lib/ceph) 2023-11-16T13:17:13.180 DEBUG:teuthology.orchestra.run.smithi167:> test -z $(ls -A /var/lib/ceph) 2023-11-16T13:17:13.213 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-11-16T13:17:13.282 INFO:teuthology.run_tasks:Running task kernel... 2023-11-16T13:17:13.298 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-11-16T13:17:13.298 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}, 'host.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-11-16T13:17:13.298 DEBUG:teuthology.orchestra.run.smithi151:> test -f /run/.containerenv -o -f /.dockerenv 2023-11-16T13:17:13.303 DEBUG:teuthology.orchestra.run.smithi167:> test -f /run/.containerenv -o -f /.dockerenv 2023-11-16T13:17:13.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-16T13:17:13.324 DEBUG:teuthology.orchestra.run.smithi167:> uname -r 2023-11-16T13:17:13.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-16T13:17:13.325 DEBUG:teuthology.orchestra.run.smithi151:> uname -r 2023-11-16T13:17:13.385 INFO:teuthology.orchestra.run.smithi167.stdout:4.18.0-489.el8.x86_64 2023-11-16T13:17:13.385 INFO:teuthology.task.kernel:Running kernel on smithi167: 4.18.0-489.el8.x86_64 2023-11-16T13:17:13.385 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum install -y kernel 2023-11-16T13:17:13.387 INFO:teuthology.orchestra.run.smithi151.stdout:4.18.0-489.el8.x86_64 2023-11-16T13:17:13.387 INFO:teuthology.task.kernel:Running kernel on smithi151: 4.18.0-489.el8.x86_64 2023-11-16T13:17:13.387 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum install -y kernel 2023-11-16T13:17:24.885 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 8 - AppStream 3.8 MB/s | 34 MB 00:09 2023-11-16T13:17:25.350 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 8 - AppStream 3.7 MB/s | 34 MB 00:09 2023-11-16T13:17:33.411 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 8 - BaseOS 22 MB/s | 53 MB 00:02 2023-11-16T13:17:33.968 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 8 - BaseOS 22 MB/s | 53 MB 00:02 2023-11-16T13:17:41.510 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 8 - Extras 31 kB/s | 18 kB 00:00 2023-11-16T13:17:41.634 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 8 - Extras 117 kB/s | 18 kB 00:00 2023-11-16T13:17:41.911 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 8 - Extras common packages 31 kB/s | 6.9 kB 00:00 2023-11-16T13:17:42.018 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 8 - Extras common packages 33 kB/s | 6.9 kB 00:00 2023-11-16T13:17:42.187 INFO:teuthology.orchestra.run.smithi167.stdout:Copr repo for python3-asyncssh owned by ceph 27 kB/s | 3.5 kB 00:00 2023-11-16T13:17:42.287 INFO:teuthology.orchestra.run.smithi151.stdout:Copr repo for python3-asyncssh owned by ceph 29 kB/s | 3.5 kB 00:00 2023-11-16T13:17:43.156 INFO:teuthology.orchestra.run.smithi151.stdout:Extra Packages for Enterprise Linux 22 MB/s | 16 MB 00:00 2023-11-16T13:17:43.213 INFO:teuthology.orchestra.run.smithi167.stdout:Extra Packages for Enterprise Linux 18 MB/s | 16 MB 00:00 2023-11-16T13:17:46.937 INFO:teuthology.orchestra.run.smithi151.stdout:lab-extras 437 kB/s | 24 kB 00:00 2023-11-16T13:17:46.980 INFO:teuthology.orchestra.run.smithi167.stdout:lab-extras 548 kB/s | 24 kB 00:00 2023-11-16T13:17:50.555 INFO:teuthology.orchestra.run.smithi167.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-11-16T13:17:50.555 INFO:teuthology.orchestra.run.smithi167.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-11-16T13:17:50.587 INFO:teuthology.orchestra.run.smithi151.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-11-16T13:17:50.587 INFO:teuthology.orchestra.run.smithi151.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-11-16T13:17:50.826 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-11-16T13:17:50.827 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-11-16T13:17:50.827 INFO:teuthology.orchestra.run.smithi167.stdout: Package Architecture Version Repository Size 2023-11-16T13:17:50.827 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-11-16T13:17:50.827 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2023-11-16T13:17:50.827 INFO:teuthology.orchestra.run.smithi167.stdout: kernel x86_64 4.18.0-521.el8 baseos 10 M 2023-11-16T13:17:50.827 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2023-11-16T13:17:50.827 INFO:teuthology.orchestra.run.smithi167.stdout: kernel-core x86_64 4.18.0-521.el8 baseos 43 M 2023-11-16T13:17:50.827 INFO:teuthology.orchestra.run.smithi167.stdout: kernel-modules x86_64 4.18.0-521.el8 baseos 36 M 2023-11-16T13:17:50.827 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:17:50.827 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-11-16T13:17:50.828 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-11-16T13:17:50.828 INFO:teuthology.orchestra.run.smithi167.stdout:Install 3 Packages 2023-11-16T13:17:50.828 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:17:50.828 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 89 M 2023-11-16T13:17:50.828 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 96 M 2023-11-16T13:17:50.828 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-11-16T13:17:50.873 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-11-16T13:17:50.873 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-11-16T13:17:50.874 INFO:teuthology.orchestra.run.smithi151.stdout: Package Architecture Version Repository Size 2023-11-16T13:17:50.874 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-11-16T13:17:50.874 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-11-16T13:17:50.874 INFO:teuthology.orchestra.run.smithi151.stdout: kernel x86_64 4.18.0-521.el8 baseos 10 M 2023-11-16T13:17:50.874 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2023-11-16T13:17:50.874 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-core x86_64 4.18.0-521.el8 baseos 43 M 2023-11-16T13:17:50.874 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-modules x86_64 4.18.0-521.el8 baseos 36 M 2023-11-16T13:17:50.874 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:17:50.874 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-11-16T13:17:50.874 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-11-16T13:17:50.875 INFO:teuthology.orchestra.run.smithi151.stdout:Install 3 Packages 2023-11-16T13:17:50.875 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:17:50.875 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 89 M 2023-11-16T13:17:50.875 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 96 M 2023-11-16T13:17:50.875 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-11-16T13:17:52.023 INFO:teuthology.orchestra.run.smithi167.stdout:(1/3): kernel-4.18.0-521.el8.x86_64.rpm 8.7 MB/s | 10 MB 00:01 2023-11-16T13:17:53.965 INFO:teuthology.orchestra.run.smithi167.stdout:(2/3): kernel-core-4.18.0-521.el8.x86_64.rpm 14 MB/s | 43 MB 00:03 2023-11-16T13:17:54.315 INFO:teuthology.orchestra.run.smithi151.stdout:(1/3): kernel-4.18.0-521.el8.x86_64.rpm 3.1 MB/s | 10 MB 00:03 2023-11-16T13:17:54.630 INFO:teuthology.orchestra.run.smithi151.stdout:(2/3): kernel-modules-4.18.0-521.el8.x86_64.rpm 10 MB/s | 36 MB 00:03 2023-11-16T13:17:54.682 INFO:teuthology.orchestra.run.smithi167.stdout:(3/3): kernel-modules-4.18.0-521.el8.x86_64.rpm 9.5 MB/s | 36 MB 00:03 2023-11-16T13:17:54.682 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-11-16T13:17:54.682 INFO:teuthology.orchestra.run.smithi167.stdout:Total 23 MB/s | 89 MB 00:03 2023-11-16T13:17:55.364 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-11-16T13:17:55.773 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-11-16T13:17:55.774 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-11-16T13:17:57.005 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-11-16T13:17:57.035 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-11-16T13:17:58.755 INFO:teuthology.orchestra.run.smithi151.stdout:(3/3): kernel-core-4.18.0-521.el8.x86_64.rpm 5.6 MB/s | 43 MB 00:07 2023-11-16T13:17:58.755 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-11-16T13:17:58.755 INFO:teuthology.orchestra.run.smithi151.stdout:Total 11 MB/s | 89 MB 00:07 2023-11-16T13:17:59.426 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-11-16T13:17:59.758 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-11-16T13:17:59.843 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-11-16T13:17:59.843 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-11-16T13:18:00.340 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : kernel-core-4.18.0-521.el8.x86_64 1/3 2023-11-16T13:18:01.060 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-11-16T13:18:01.093 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-11-16T13:18:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-11-16T13:18:03.872 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: kernel-core-4.18.0-521.el8.x86_64 1/3 2023-11-16T13:18:04.248 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : kernel-modules-4.18.0-521.el8.x86_64 2/3 2023-11-16T13:18:04.533 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : kernel-core-4.18.0-521.el8.x86_64 1/3 2023-11-16T13:18:07.692 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: kernel-modules-4.18.0-521.el8.x86_64 2/3 2023-11-16T13:18:07.996 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : kernel-4.18.0-521.el8.x86_64 3/3 2023-11-16T13:18:08.101 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-core-4.18.0-521.el8.x86_64 1/3 2023-11-16T13:18:08.515 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : kernel-modules-4.18.0-521.el8.x86_64 2/3 2023-11-16T13:18:11.938 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-modules-4.18.0-521.el8.x86_64 2/3 2023-11-16T13:18:12.211 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : kernel-4.18.0-521.el8.x86_64 3/3 2023-11-16T13:18:29.824 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: kernel-core-4.18.0-521.el8.x86_64 3/3 2023-11-16T13:18:29.848 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: kernel-modules-4.18.0-521.el8.x86_64 3/3 2023-11-16T13:18:30.208 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: kernel-4.18.0-521.el8.x86_64 3/3 2023-11-16T13:18:30.208 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : kernel-4.18.0-521.el8.x86_64 1/3 2023-11-16T13:18:30.208 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : kernel-core-4.18.0-521.el8.x86_64 2/3 2023-11-16T13:18:31.226 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : kernel-modules-4.18.0-521.el8.x86_64 3/3 2023-11-16T13:18:31.226 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:18:31.226 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-11-16T13:18:31.226 INFO:teuthology.orchestra.run.smithi167.stdout: kernel-4.18.0-521.el8.x86_64 kernel-core-4.18.0-521.el8.x86_64 2023-11-16T13:18:31.226 INFO:teuthology.orchestra.run.smithi167.stdout: kernel-modules-4.18.0-521.el8.x86_64 2023-11-16T13:18:31.227 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:18:31.227 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-11-16T13:18:31.378 DEBUG:teuthology.orchestra.run.smithi167:> rpm -q kernel | sort -rV | head -n 1 2023-11-16T13:18:31.595 INFO:teuthology.orchestra.run.smithi167.stdout:kernel-4.18.0-521.el8.x86_64 2023-11-16T13:18:31.595 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-521.el8.x86_64 2023-11-16T13:18:31.595 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-521.el8.x86_64 2023-11-16T13:18:31.596 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-11-16T13:18:31.596 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-11-16T13:18:31.596 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2023-11-16T13:18:31.596 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi167.front.sepia.ceph.com, path=None, version=distro) 2023-11-16T13:18:31.596 DEBUG:teuthology.orchestra.run.smithi167:> rpm -q kernel | sort -rV | head -n 1 2023-11-16T13:18:31.806 INFO:teuthology.orchestra.run.smithi167.stdout:kernel-4.18.0-521.el8.x86_64 2023-11-16T13:18:31.807 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-521.el8.x86_64 2023-11-16T13:18:31.807 DEBUG:teuthology.orchestra.run.smithi167:> sudo rpm -qi grub2-tools 2023-11-16T13:18:31.840 INFO:teuthology.orchestra.run.smithi167.stdout:Name : grub2-tools 2023-11-16T13:18:31.840 INFO:teuthology.orchestra.run.smithi167.stdout:Epoch : 1 2023-11-16T13:18:31.840 INFO:teuthology.orchestra.run.smithi167.stdout:Version : 2.02 2023-11-16T13:18:31.840 INFO:teuthology.orchestra.run.smithi167.stdout:Release : 129.el8 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:Architecture: x86_64 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:Group : System Environment/Base 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:Size : 9265586 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:License : GPLv3+ 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:Build Host : kbuilder.bsys.centos.org 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:Relocations : (not relocatable) 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:Packager : CentOS BuildSystem 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:Vendor : CentOS 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:URL : http://www.gnu.org/software/grub/ 2023-11-16T13:18:31.841 INFO:teuthology.orchestra.run.smithi167.stdout:Summary : Support tools for GRUB. 2023-11-16T13:18:31.842 INFO:teuthology.orchestra.run.smithi167.stdout:Description : 2023-11-16T13:18:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:18:31.842 INFO:teuthology.orchestra.run.smithi167.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-11-16T13:18:31.842 INFO:teuthology.orchestra.run.smithi167.stdout:customizable bootloader with modular architecture. It supports a rich 2023-11-16T13:18:31.842 INFO:teuthology.orchestra.run.smithi167.stdout:variety of kernel formats, file systems, computer architectures and 2023-11-16T13:18:31.842 INFO:teuthology.orchestra.run.smithi167.stdout:hardware devices. 2023-11-16T13:18:31.842 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:18:31.842 INFO:teuthology.orchestra.run.smithi167.stdout:This subpackage provides tools for support of all platforms. 2023-11-16T13:18:31.843 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-11-16T13:18:31.843 INFO:teuthology.task.kernel:Updating grub on smithi167 to boot 4.18.0-521.el8.x86_64 2023-11-16T13:18:31.843 DEBUG:teuthology.orchestra.run.smithi167:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-11-16T13:18:33.361 INFO:teuthology.orchestra.run.smithi167.stderr:Generating grub configuration file ... 2023-11-16T13:18:34.190 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-core-4.18.0-521.el8.x86_64 3/3 2023-11-16T13:18:34.213 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-modules-4.18.0-521.el8.x86_64 3/3 2023-11-16T13:18:34.588 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-4.18.0-521.el8.x86_64 3/3 2023-11-16T13:18:34.588 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : kernel-4.18.0-521.el8.x86_64 1/3 2023-11-16T13:18:34.588 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : kernel-core-4.18.0-521.el8.x86_64 2/3 2023-11-16T13:18:35.191 INFO:teuthology.orchestra.run.smithi167.stderr:done 2023-11-16T13:18:35.193 DEBUG:teuthology.orchestra.run.smithi167:> mktemp 2023-11-16T13:18:35.215 INFO:teuthology.orchestra.run.smithi167.stdout:/tmp/tmp.a5rJoRAqYo 2023-11-16T13:18:35.216 DEBUG:teuthology.orchestra.run.smithi167:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.a5rJoRAqYo 2023-11-16T13:18:35.290 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod 0666 /tmp/tmp.a5rJoRAqYo 2023-11-16T13:18:35.394 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : kernel-modules-4.18.0-521.el8.x86_64 3/3 2023-11-16T13:18:35.394 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:18:35.394 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-11-16T13:18:35.394 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-4.18.0-521.el8.x86_64 kernel-core-4.18.0-521.el8.x86_64 2023-11-16T13:18:35.394 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-modules-4.18.0-521.el8.x86_64 2023-11-16T13:18:35.394 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:18:35.394 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-11-16T13:18:35.427 DEBUG:teuthology.orchestra.remote:smithi167:/tmp/tmp.a5rJoRAqYo is 6KB 2023-11-16T13:18:35.475 DEBUG:teuthology.orchestra.run.smithi167:> rm -fr /tmp/tmp.a5rJoRAqYo 2023-11-16T13:18:35.493 DEBUG:teuthology.orchestra.run.smithi167:> sudo /bin/ls /boot/loader/entries || true 2023-11-16T13:18:35.537 DEBUG:teuthology.orchestra.run.smithi151:> rpm -q kernel | sort -rV | head -n 1 2023-11-16T13:18:35.558 INFO:teuthology.orchestra.run.smithi167.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-11-16T13:18:35.558 INFO:teuthology.orchestra.run.smithi167.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-11-16T13:18:35.558 INFO:teuthology.orchestra.run.smithi167.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-11-16T13:18:35.559 INFO:teuthology.orchestra.run.smithi167.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-521.el8.x86_64.conf 2023-11-16T13:18:35.559 DEBUG:teuthology.orchestra.run.smithi167:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-521.el8.x86_64 2023-11-16T13:18:35.729 DEBUG:teuthology.orchestra.run.smithi167:> sudo shutdown -r now 2023-11-16T13:18:35.751 INFO:teuthology.orchestra.run.smithi151.stdout:kernel-4.18.0-521.el8.x86_64 2023-11-16T13:18:35.752 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-521.el8.x86_64 2023-11-16T13:18:35.752 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-521.el8.x86_64 2023-11-16T13:18:35.752 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-11-16T13:18:35.752 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-11-16T13:18:35.752 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2023-11-16T13:18:35.752 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi151.front.sepia.ceph.com, path=None, version=distro) 2023-11-16T13:18:35.752 DEBUG:teuthology.orchestra.run.smithi151:> rpm -q kernel | sort -rV | head -n 1 2023-11-16T13:18:35.963 INFO:teuthology.orchestra.run.smithi151.stdout:kernel-4.18.0-521.el8.x86_64 2023-11-16T13:18:35.963 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-521.el8.x86_64 2023-11-16T13:18:35.964 DEBUG:teuthology.orchestra.run.smithi151:> sudo rpm -qi grub2-tools 2023-11-16T13:18:35.997 INFO:teuthology.orchestra.run.smithi151.stdout:Name : grub2-tools 2023-11-16T13:18:35.997 INFO:teuthology.orchestra.run.smithi151.stdout:Epoch : 1 2023-11-16T13:18:35.997 INFO:teuthology.orchestra.run.smithi151.stdout:Version : 2.02 2023-11-16T13:18:35.997 INFO:teuthology.orchestra.run.smithi151.stdout:Release : 129.el8 2023-11-16T13:18:35.997 INFO:teuthology.orchestra.run.smithi151.stdout:Architecture: x86_64 2023-11-16T13:18:35.997 INFO:teuthology.orchestra.run.smithi151.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-11-16T13:18:35.998 INFO:teuthology.orchestra.run.smithi151.stdout:Group : System Environment/Base 2023-11-16T13:18:35.998 INFO:teuthology.orchestra.run.smithi151.stdout:Size : 9265586 2023-11-16T13:18:35.998 INFO:teuthology.orchestra.run.smithi151.stdout:License : GPLv3+ 2023-11-16T13:18:35.998 INFO:teuthology.orchestra.run.smithi151.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-11-16T13:18:35.998 INFO:teuthology.orchestra.run.smithi151.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-11-16T13:18:35.998 INFO:teuthology.orchestra.run.smithi151.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-11-16T13:18:35.998 INFO:teuthology.orchestra.run.smithi151.stdout:Build Host : kbuilder.bsys.centos.org 2023-11-16T13:18:35.998 INFO:teuthology.orchestra.run.smithi151.stdout:Relocations : (not relocatable) 2023-11-16T13:18:35.998 INFO:teuthology.orchestra.run.smithi151.stdout:Packager : CentOS BuildSystem 2023-11-16T13:18:35.998 INFO:teuthology.orchestra.run.smithi151.stdout:Vendor : CentOS 2023-11-16T13:18:35.998 INFO:teuthology.orchestra.run.smithi151.stdout:URL : http://www.gnu.org/software/grub/ 2023-11-16T13:18:35.998 INFO:teuthology.orchestra.run.smithi151.stdout:Summary : Support tools for GRUB. 2023-11-16T13:18:35.998 INFO:teuthology.orchestra.run.smithi151.stdout:Description : 2023-11-16T13:18:35.999 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:18:35.999 INFO:teuthology.orchestra.run.smithi151.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-11-16T13:18:35.999 INFO:teuthology.orchestra.run.smithi151.stdout:customizable bootloader with modular architecture. It supports a rich 2023-11-16T13:18:35.999 INFO:teuthology.orchestra.run.smithi151.stdout:variety of kernel formats, file systems, computer architectures and 2023-11-16T13:18:35.999 INFO:teuthology.orchestra.run.smithi151.stdout:hardware devices. 2023-11-16T13:18:35.999 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:18:35.999 INFO:teuthology.orchestra.run.smithi151.stdout:This subpackage provides tools for support of all platforms. 2023-11-16T13:18:36.000 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-11-16T13:18:36.000 INFO:teuthology.task.kernel:Updating grub on smithi151 to boot 4.18.0-521.el8.x86_64 2023-11-16T13:18:36.000 DEBUG:teuthology.orchestra.run.smithi151:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-11-16T13:18:37.560 INFO:teuthology.orchestra.run.smithi151.stderr:Generating grub configuration file ... 2023-11-16T13:18:39.408 INFO:teuthology.orchestra.run.smithi151.stderr:done 2023-11-16T13:18:39.410 DEBUG:teuthology.orchestra.run.smithi151:> mktemp 2023-11-16T13:18:39.451 INFO:teuthology.orchestra.run.smithi151.stdout:/tmp/tmp.brJQA9sZZD 2023-11-16T13:18:39.451 DEBUG:teuthology.orchestra.run.smithi151:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.brJQA9sZZD 2023-11-16T13:18:39.523 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod 0666 /tmp/tmp.brJQA9sZZD 2023-11-16T13:18:39.664 DEBUG:teuthology.orchestra.remote:smithi151:/tmp/tmp.brJQA9sZZD is 6KB 2023-11-16T13:18:39.711 DEBUG:teuthology.orchestra.run.smithi151:> rm -fr /tmp/tmp.brJQA9sZZD 2023-11-16T13:18:39.729 DEBUG:teuthology.orchestra.run.smithi151:> sudo /bin/ls /boot/loader/entries || true 2023-11-16T13:18:39.798 INFO:teuthology.orchestra.run.smithi151.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-11-16T13:18:39.798 INFO:teuthology.orchestra.run.smithi151.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-11-16T13:18:39.798 INFO:teuthology.orchestra.run.smithi151.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-11-16T13:18:39.799 INFO:teuthology.orchestra.run.smithi151.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-521.el8.x86_64.conf 2023-11-16T13:18:39.799 DEBUG:teuthology.orchestra.run.smithi151:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-521.el8.x86_64 2023-11-16T13:18:39.977 DEBUG:teuthology.orchestra.run.smithi151:> sudo shutdown -r now 2023-11-16T13:19:05.758 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2023-11-16T13:19:05.759 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi167.front.sepia.ceph.com' 2023-11-16T13:19:05.760 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-16T13:19:09.982 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2023-11-16T13:19:09.982 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi151.front.sepia.ceph.com' 2023-11-16T13:19:09.983 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-16T13:19:24.286 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.167 2023-11-16T13:19:28.378 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.151 2023-11-16T13:19:33.289 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi167.front.sepia.ceph.com' 2023-11-16T13:19:33.290 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-16T13:19:36.346 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.167 2023-11-16T13:19:40.436 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi151.front.sepia.ceph.com' 2023-11-16T13:19:40.436 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-16T13:19:43.514 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.151 2023-11-16T13:19:48.350 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi167.front.sepia.ceph.com' 2023-11-16T13:19:48.351 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-16T13:19:55.522 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi151.front.sepia.ceph.com' 2023-11-16T13:19:55.523 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-16T13:20:06.778 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.167 2023-11-16T13:20:21.793 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi167.front.sepia.ceph.com' 2023-11-16T13:20:21.795 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-16T13:20:22.191 DEBUG:teuthology.orchestra.run.smithi167:> true 2023-11-16T13:20:22.472 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi167.front.sepia.ceph.com' 2023-11-16T13:20:22.472 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "4.18.0-521.el8.x86_64"... 2023-11-16T13:20:22.473 DEBUG:teuthology.orchestra.run.smithi167:> uname -r 2023-11-16T13:20:22.489 INFO:teuthology.orchestra.run.smithi167.stdout:4.18.0-521.el8.x86_64 2023-11-16T13:20:22.489 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-521.el8.x86_64 vs 4.18.0-521.el8.x86_64 2023-11-16T13:20:22.490 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-11-16T13:20:22.491 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-11-16T13:20:23.491 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2023-11-16T13:20:23.492 DEBUG:teuthology.orchestra.run.smithi167:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-11-16T13:20:23.568 INFO:teuthology.orchestra.run.smithi167.stdout:ttyS1 2023-11-16T13:20:23.586 DEBUG:teuthology.parallel:result is None 2023-11-16T13:20:27.356 DEBUG:teuthology.orchestra.run.smithi151:> true 2023-11-16T13:20:27.693 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi151.front.sepia.ceph.com' 2023-11-16T13:20:27.693 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "4.18.0-521.el8.x86_64"... 2023-11-16T13:20:27.693 DEBUG:teuthology.orchestra.run.smithi151:> uname -r 2023-11-16T13:20:27.710 INFO:teuthology.orchestra.run.smithi151.stdout:4.18.0-521.el8.x86_64 2023-11-16T13:20:27.710 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-521.el8.x86_64 vs 4.18.0-521.el8.x86_64 2023-11-16T13:20:27.710 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-11-16T13:20:27.710 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-11-16T13:20:28.711 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2023-11-16T13:20:28.711 DEBUG:teuthology.orchestra.run.smithi151:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-11-16T13:20:28.761 INFO:teuthology.orchestra.run.smithi151.stdout:ttyS1 2023-11-16T13:20:28.783 DEBUG:teuthology.parallel:result is None 2023-11-16T13:20:28.783 INFO:teuthology.run_tasks:Running task internal.base... 2023-11-16T13:20:28.792 INFO:teuthology.task.internal:Creating test directory... 2023-11-16T13:20:28.793 DEBUG:teuthology.orchestra.run.smithi151:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-11-16T13:20:28.796 DEBUG:teuthology.orchestra.run.smithi167:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-11-16T13:20:28.820 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-11-16T13:20:28.835 INFO:teuthology.run_tasks:Running task internal.archive... 2023-11-16T13:20:28.869 INFO:teuthology.task.internal:Creating archive directory... 2023-11-16T13:20:28.869 DEBUG:teuthology.orchestra.run.smithi151:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-11-16T13:20:28.872 DEBUG:teuthology.orchestra.run.smithi167:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-11-16T13:20:28.907 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-11-16T13:20:28.917 INFO:teuthology.task.internal:Enabling coredump saving... 2023-11-16T13:20:28.917 DEBUG:teuthology.orchestra.run.smithi151:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-11-16T13:20:28.933 DEBUG:teuthology.orchestra.run.smithi167:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-11-16T13:20:28.970 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-11-16T13:20:28.981 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-11-16T13:20:28.997 INFO:teuthology.orchestra.run.smithi167.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-11-16T13:20:29.007 INFO:teuthology.orchestra.run.smithi167.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-11-16T13:20:29.009 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-11-16T13:20:29.017 INFO:teuthology.task.internal:Configuring sudo... 2023-11-16T13:20:29.018 DEBUG:teuthology.orchestra.run.smithi151:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-11-16T13:20:29.030 DEBUG:teuthology.orchestra.run.smithi167:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-11-16T13:20:29.060 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-11-16T13:20:29.073 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-11-16T13:20:29.074 DEBUG:teuthology.orchestra.run.smithi151:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-11-16T13:20:29.103 DEBUG:teuthology.orchestra.run.smithi167:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-11-16T13:20:29.123 DEBUG:teuthology.orchestra.run.smithi151:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-11-16T13:20:29.209 DEBUG:teuthology.orchestra.run.smithi151:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-11-16T13:20:29.263 DEBUG:teuthology.orchestra.run.smithi151:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-11-16T13:20:29.349 DEBUG:teuthology.orchestra.run.smithi151:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-11-16T13:20:29.395 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-11-16T13:20:29.395 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-11-16T13:20:29.465 DEBUG:teuthology.orchestra.run.smithi167:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-11-16T13:20:29.512 DEBUG:teuthology.orchestra.run.smithi167:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-11-16T13:20:29.567 DEBUG:teuthology.orchestra.run.smithi167:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-11-16T13:20:29.658 DEBUG:teuthology.orchestra.run.smithi167:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-11-16T13:20:29.706 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-11-16T13:20:29.706 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-11-16T13:20:29.778 DEBUG:teuthology.orchestra.run.smithi151:> sudo service rsyslog restart 2023-11-16T13:20:29.781 DEBUG:teuthology.orchestra.run.smithi167:> sudo service rsyslog restart 2023-11-16T13:20:29.842 INFO:teuthology.orchestra.run.smithi151.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-11-16T13:20:29.895 INFO:teuthology.orchestra.run.smithi167.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-11-16T13:20:30.393 INFO:teuthology.run_tasks:Running task internal.timer... 2023-11-16T13:20:30.400 INFO:teuthology.task.internal:Starting timer... 2023-11-16T13:20:30.401 INFO:teuthology.run_tasks:Running task pcp... 2023-11-16T13:20:30.446 INFO:teuthology.run_tasks:Running task selinux... 2023-11-16T13:20:30.477 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-11-16T13:20:30.478 DEBUG:teuthology.orchestra.run.smithi151:> sudo service auditd rotate 2023-11-16T13:20:30.620 INFO:teuthology.orchestra.run.smithi151.stdout:Rotating logs: 2023-11-16T13:20:30.623 DEBUG:teuthology.orchestra.run.smithi167:> sudo service auditd rotate 2023-11-16T13:20:30.716 INFO:teuthology.orchestra.run.smithi167.stdout:Rotating logs: 2023-11-16T13:20:30.718 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-11-16T13:20:30.718 DEBUG:teuthology.orchestra.run.smithi151:> /usr/sbin/getenforce 2023-11-16T13:20:30.856 INFO:teuthology.orchestra.run.smithi151.stdout:Permissive 2023-11-16T13:20:30.857 DEBUG:teuthology.orchestra.run.smithi167:> /usr/sbin/getenforce 2023-11-16T13:20:30.884 INFO:teuthology.orchestra.run.smithi167.stdout:Permissive 2023-11-16T13:20:30.885 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi151.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi167.front.sepia.ceph.com': 'permissive'} 2023-11-16T13:20:30.885 DEBUG:teuthology.orchestra.run.smithi151:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-11-16T13:20:31.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-16T13:20:31.306 DEBUG:teuthology.orchestra.run.smithi167:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-11-16T13:20:31.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-16T13:20:31.333 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-11-16T13:20:31.334 DEBUG:teuthology.orchestra.run.smithi151:> sudo /usr/sbin/setenforce permissive 2023-11-16T13:20:31.851 DEBUG:teuthology.orchestra.run.smithi167:> sudo /usr/sbin/setenforce permissive 2023-11-16T13:20:31.885 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-11-16T13:20:31.895 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current 2023-11-16T13:20:31.895 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-11-16T13:20:31.912 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-11-16T13:20:31.913 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi151.front.sepia.ceph.com,smithi167.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-11-16T13:28:15.991 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi151.front.sepia.ceph.com'), Remote(name='ubuntu@smithi167.front.sepia.ceph.com')] 2023-11-16T13:28:15.992 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi151.front.sepia.ceph.com' 2023-11-16T13:28:15.993 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-16T13:28:16.060 DEBUG:teuthology.orchestra.run.smithi151:> true 2023-11-16T13:28:16.131 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi151.front.sepia.ceph.com' 2023-11-16T13:28:16.132 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi167.front.sepia.ceph.com' 2023-11-16T13:28:16.132 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-16T13:28:16.201 DEBUG:teuthology.orchestra.run.smithi167:> true 2023-11-16T13:28:16.274 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi167.front.sepia.ceph.com' 2023-11-16T13:28:16.274 INFO:teuthology.run_tasks:Running task clock... 2023-11-16T13:28:16.285 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-11-16T13:28:16.286 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-11-16T13:28:16.286 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-11-16T13:28:16.288 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-11-16T13:28:16.288 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-11-16T13:28:16.317 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-11-16T13:28:16.330 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-11-16T13:28:16.348 INFO:teuthology.orchestra.run.smithi167.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-11-16T13:28:16.362 INFO:teuthology.orchestra.run.smithi167.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-11-16T13:28:16.389 INFO:teuthology.orchestra.run.smithi151.stderr:sudo: ntpd: command not found 2023-11-16T13:28:16.401 INFO:teuthology.orchestra.run.smithi151.stdout:506 Cannot talk to daemon 2023-11-16T13:28:16.415 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-11-16T13:28:16.428 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-11-16T13:28:16.431 INFO:teuthology.orchestra.run.smithi167.stderr:sudo: ntpd: command not found 2023-11-16T13:28:16.442 INFO:teuthology.orchestra.run.smithi167.stdout:506 Cannot talk to daemon 2023-11-16T13:28:16.456 INFO:teuthology.orchestra.run.smithi167.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-11-16T13:28:16.469 INFO:teuthology.orchestra.run.smithi167.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-11-16T13:28:16.496 INFO:teuthology.orchestra.run.smithi151.stderr:bash: ntpq: command not found 2023-11-16T13:28:16.499 INFO:teuthology.orchestra.run.smithi151.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-11-16T13:28:16.499 INFO:teuthology.orchestra.run.smithi151.stdout:=============================================================================== 2023-11-16T13:28:16.500 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-16T13:28:16.500 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-16T13:28:16.500 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-16T13:28:16.500 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-16T13:28:16.544 INFO:teuthology.orchestra.run.smithi167.stderr:bash: ntpq: command not found 2023-11-16T13:28:16.547 INFO:teuthology.orchestra.run.smithi167.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-11-16T13:28:16.547 INFO:teuthology.orchestra.run.smithi167.stdout:=============================================================================== 2023-11-16T13:28:16.547 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-16T13:28:16.548 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-16T13:28:16.548 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-16T13:28:16.548 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-16T13:28:16.549 INFO:teuthology.run_tasks:Running task pexec... 2023-11-16T13:28:16.558 INFO:teuthology.task.pexec:Executing custom commands... 2023-11-16T13:28:16.558 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi151.front.sepia.ceph.com 2023-11-16T13:28:16.558 DEBUG:teuthology.orchestra.run.smithi151:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-11-16T13:28:16.559 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi167.front.sepia.ceph.com 2023-11-16T13:28:16.559 DEBUG:teuthology.orchestra.run.smithi167:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-11-16T13:28:17.442 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:54 ago on Thu 16 Nov 2023 01:26:23 PM UTC. 2023-11-16T13:28:17.475 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:55 ago on Thu 16 Nov 2023 01:26:22 PM UTC. 2023-11-16T13:28:19.547 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-11-16T13:28:19.548 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-11-16T13:28:19.548 INFO:teuthology.orchestra.run.smithi151.stdout: Package Architecture Version Repository Size 2023-11-16T13:28:19.548 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-11-16T13:28:19.548 INFO:teuthology.orchestra.run.smithi151.stdout:Resetting modules: 2023-11-16T13:28:19.548 INFO:teuthology.orchestra.run.smithi151.stdout: container-tools 2023-11-16T13:28:19.549 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:28:19.549 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-11-16T13:28:19.549 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-11-16T13:28:19.549 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:28:19.612 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-11-16T13:28:19.613 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-11-16T13:28:19.613 INFO:teuthology.orchestra.run.smithi167.stdout: Package Architecture Version Repository Size 2023-11-16T13:28:19.613 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-11-16T13:28:19.613 INFO:teuthology.orchestra.run.smithi167.stdout:Resetting modules: 2023-11-16T13:28:19.613 INFO:teuthology.orchestra.run.smithi167.stdout: container-tools 2023-11-16T13:28:19.613 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:28:19.613 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-11-16T13:28:19.613 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-11-16T13:28:19.613 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:28:19.799 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-11-16T13:28:19.868 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-11-16T13:28:20.773 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:57 ago on Thu 16 Nov 2023 01:26:23 PM UTC. 2023-11-16T13:28:20.818 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:58 ago on Thu 16 Nov 2023 01:26:22 PM UTC. 2023-11-16T13:28:22.891 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-11-16T13:28:22.913 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2023-11-16T13:28:23.068 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-11-16T13:28:23.068 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2023-11-16T13:28:23.068 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-11-16T13:28:23.068 INFO:teuthology.orchestra.run.smithi151.stdout:Installing group/module packages: 2023-11-16T13:28:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: aardvark-dns x86_64 2:1.8.0-1.module_el8+712+4cd1bd69 appstream 1.0 M 2023-11-16T13:28:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: buildah x86_64 1:1.32.0-1.module_el8+712+4cd1bd69 appstream 9.6 M 2023-11-16T13:28:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: cockpit-podman noarch 78-1.module_el8+712+4cd1bd69 appstream 750 k 2023-11-16T13:28:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: crun x86_64 1.9.2-1.module_el8+712+4cd1bd69 appstream 241 k 2023-11-16T13:28:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: netavark x86_64 2:1.8.0-2.module_el8+712+4cd1bd69 appstream 3.9 M 2023-11-16T13:28:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: python3-podman noarch 4.7.0-1.module_el8+712+4cd1bd69 appstream 178 k 2023-11-16T13:28:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: skopeo x86_64 2:1.13.3-1.module_el8+712+4cd1bd69 appstream 8.1 M 2023-11-16T13:28:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: toolbox x86_64 0.0.99.4-6.module_el8+712+4cd1bd69 appstream 2.5 M 2023-11-16T13:28:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: udica noarch 0.2.6-20.module_el8+712+4cd1bd69 appstream 53 k 2023-11-16T13:28:23.069 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2023-11-16T13:28:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2023-11-16T13:28:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2023-11-16T13:28:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2023-11-16T13:28:23.070 INFO:teuthology.orchestra.run.smithi151.stdout:Installing weak dependencies: 2023-11-16T13:28:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2023-11-16T13:28:23.070 INFO:teuthology.orchestra.run.smithi151.stdout:Installing module profiles: 2023-11-16T13:28:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: container-tools/common 2023-11-16T13:28:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:28:23.070 INFO:teuthology.orchestra.run.smithi151.stdout:Enabling module streams: 2023-11-16T13:28:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: container-tools rhel8 2023-11-16T13:28:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:28:23.070 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-11-16T13:28:23.070 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-11-16T13:28:23.071 INFO:teuthology.orchestra.run.smithi151.stdout:Install 13 Packages 2023-11-16T13:28:23.071 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:28:23.071 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 27 M 2023-11-16T13:28:23.071 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 89 M 2023-11-16T13:28:23.071 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-11-16T13:28:23.092 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-11-16T13:28:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2023-11-16T13:28:23.092 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-11-16T13:28:23.092 INFO:teuthology.orchestra.run.smithi167.stdout:Installing group/module packages: 2023-11-16T13:28:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: aardvark-dns x86_64 2:1.8.0-1.module_el8+712+4cd1bd69 appstream 1.0 M 2023-11-16T13:28:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: buildah x86_64 1:1.32.0-1.module_el8+712+4cd1bd69 appstream 9.6 M 2023-11-16T13:28:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: cockpit-podman noarch 78-1.module_el8+712+4cd1bd69 appstream 750 k 2023-11-16T13:28:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: crun x86_64 1.9.2-1.module_el8+712+4cd1bd69 appstream 241 k 2023-11-16T13:28:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: netavark x86_64 2:1.8.0-2.module_el8+712+4cd1bd69 appstream 3.9 M 2023-11-16T13:28:23.092 INFO:teuthology.orchestra.run.smithi167.stdout: python3-podman noarch 4.7.0-1.module_el8+712+4cd1bd69 appstream 178 k 2023-11-16T13:28:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: skopeo x86_64 2:1.13.3-1.module_el8+712+4cd1bd69 appstream 8.1 M 2023-11-16T13:28:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: toolbox x86_64 0.0.99.4-6.module_el8+712+4cd1bd69 appstream 2.5 M 2023-11-16T13:28:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: udica noarch 0.2.6-20.module_el8+712+4cd1bd69 appstream 53 k 2023-11-16T13:28:23.093 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2023-11-16T13:28:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2023-11-16T13:28:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2023-11-16T13:28:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2023-11-16T13:28:23.093 INFO:teuthology.orchestra.run.smithi167.stdout:Installing weak dependencies: 2023-11-16T13:28:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2023-11-16T13:28:23.093 INFO:teuthology.orchestra.run.smithi167.stdout:Installing module profiles: 2023-11-16T13:28:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: container-tools/common 2023-11-16T13:28:23.093 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:28:23.093 INFO:teuthology.orchestra.run.smithi167.stdout:Enabling module streams: 2023-11-16T13:28:23.094 INFO:teuthology.orchestra.run.smithi167.stdout: container-tools rhel8 2023-11-16T13:28:23.094 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:28:23.094 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2023-11-16T13:28:23.094 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2023-11-16T13:28:23.094 INFO:teuthology.orchestra.run.smithi167.stdout:Install 13 Packages 2023-11-16T13:28:23.094 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:28:23.095 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 27 M 2023-11-16T13:28:23.095 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 89 M 2023-11-16T13:28:23.095 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2023-11-16T13:28:24.057 INFO:teuthology.orchestra.run.smithi151.stdout:(1/13): cockpit-podman-78-1.module_el8+712+4cd1 980 kB/s | 750 kB 00:00 2023-11-16T13:28:24.149 INFO:teuthology.orchestra.run.smithi151.stdout:(2/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 414 kB/s | 38 kB 00:00 2023-11-16T13:28:24.266 INFO:teuthology.orchestra.run.smithi151.stdout:(3/13): crun-1.9.2-1.module_el8+712+4cd1bd69.x8 2.0 MB/s | 241 kB 00:00 2023-11-16T13:28:24.664 INFO:teuthology.orchestra.run.smithi167.stdout:(1/13): cockpit-podman-78-1.module_el8+712+4cd1 552 kB/s | 750 kB 00:01 2023-11-16T13:28:24.773 INFO:teuthology.orchestra.run.smithi167.stdout:(2/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 350 kB/s | 38 kB 00:00 2023-11-16T13:28:24.848 INFO:teuthology.orchestra.run.smithi167.stdout:(3/13): aardvark-dns-1.8.0-1.module_el8+712+4cd 666 kB/s | 1.0 MB 00:01 2023-11-16T13:28:25.073 INFO:teuthology.orchestra.run.smithi167.stdout:(4/13): crun-1.9.2-1.module_el8+712+4cd1bd69.x8 802 kB/s | 241 kB 00:00 2023-11-16T13:28:25.141 INFO:teuthology.orchestra.run.smithi151.stdout:(4/13): aardvark-dns-1.8.0-1.module_el8+712+4cd 555 kB/s | 1.0 MB 00:01 2023-11-16T13:28:25.359 INFO:teuthology.orchestra.run.smithi151.stdout:(5/13): python3-podman-4.7.0-1.module_el8+712+4 819 kB/s | 178 kB 00:00 2023-11-16T13:28:25.491 INFO:teuthology.orchestra.run.smithi167.stdout:(5/13): python3-podman-4.7.0-1.module_el8+712+4 426 kB/s | 178 kB 00:00 2023-11-16T13:28:25.551 INFO:teuthology.orchestra.run.smithi151.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 129 kB/s | 25 kB 00:00 2023-11-16T13:28:25.692 INFO:teuthology.orchestra.run.smithi151.stdout:(7/13): python3-pyxdg-0.25-16.el8.noarch.rpm 663 kB/s | 94 kB 00:00 2023-11-16T13:28:25.708 INFO:teuthology.orchestra.run.smithi167.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 115 kB/s | 25 kB 00:00 2023-11-16T13:28:25.891 INFO:teuthology.orchestra.run.smithi167.stdout:(7/13): python3-pyxdg-0.25-16.el8.noarch.rpm 512 kB/s | 94 kB 00:00 2023-11-16T13:28:26.676 INFO:teuthology.orchestra.run.smithi151.stdout:(8/13): netavark-1.8.0-2.module_el8+712+4cd1bd6 1.6 MB/s | 3.9 MB 00:02 2023-11-16T13:28:27.783 INFO:teuthology.orchestra.run.smithi167.stdout:(8/13): netavark-1.8.0-2.module_el8+712+4cd1bd6 1.3 MB/s | 3.9 MB 00:02 2023-11-16T13:28:29.043 INFO:teuthology.orchestra.run.smithi151.stdout:(9/13): skopeo-1.13.3-1.module_el8+712+4cd1bd69 2.4 MB/s | 8.1 MB 00:03 2023-11-16T13:28:29.118 INFO:teuthology.orchestra.run.smithi151.stdout:(10/13): toolbox-0.0.99.4-6.module_el8+712+4cd1 1.0 MB/s | 2.5 MB 00:02 2023-11-16T13:28:29.144 INFO:teuthology.orchestra.run.smithi151.stdout:(11/13): udica-0.2.6-20.module_el8+712+4cd1bd69 524 kB/s | 53 kB 00:00 2023-11-16T13:28:29.477 INFO:teuthology.orchestra.run.smithi151.stdout:(12/13): yajl-2.1.0-12.el8.x86_64.rpm 114 kB/s | 41 kB 00:00 2023-11-16T13:28:29.641 INFO:teuthology.orchestra.run.smithi167.stdout:(9/13): toolbox-0.0.99.4-6.module_el8+712+4cd1b 1.4 MB/s | 2.5 MB 00:01 2023-11-16T13:28:29.891 INFO:teuthology.orchestra.run.smithi167.stdout:(10/13): udica-0.2.6-20.module_el8+712+4cd1bd69 211 kB/s | 53 kB 00:00 2023-11-16T13:28:30.192 INFO:teuthology.orchestra.run.smithi167.stdout:(11/13): yajl-2.1.0-12.el8.x86_64.rpm 136 kB/s | 41 kB 00:00 2023-11-16T13:28:30.802 INFO:teuthology.orchestra.run.smithi151.stdout:(13/13): buildah-1.32.0-1.module_el8+712+4cd1bd 1.3 MB/s | 9.6 MB 00:07 2023-11-16T13:28:30.803 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-11-16T13:28:30.803 INFO:teuthology.orchestra.run.smithi151.stdout:Total 3.4 MB/s | 27 MB 00:07 2023-11-16T13:28:31.077 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-11-16T13:28:31.099 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-11-16T13:28:31.099 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-11-16T13:28:31.360 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-11-16T13:28:31.364 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-11-16T13:28:32.089 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-11-16T13:28:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2023-11-16T13:28:32.532 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/13 2023-11-16T13:28:32.735 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/13 2023-11-16T13:28:32.993 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-11-16T13:28:33.552 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : aardvark-dns-2:1.8.0-1.module_el8+712+4cd1bd69.x86 5/13 2023-11-16T13:28:34.557 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : netavark-2:1.8.0-2.module_el8+712+4cd1bd69.x86_64 6/13 2023-11-16T13:28:34.762 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : buildah-1:1.32.0-1.module_el8+712+4cd1bd69.x86_64 7/13 2023-11-16T13:28:34.963 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : crun-1.9.2-1.module_el8+712+4cd1bd69.x86_64 8/13 2023-11-16T13:28:35.269 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-podman-4.7.0-1.module_el8+712+4cd1bd69.noa 9/13 2023-11-16T13:28:35.300 INFO:teuthology.orchestra.run.smithi167.stdout:(12/13): skopeo-1.13.3-1.module_el8+712+4cd1bd6 886 kB/s | 8.1 MB 00:09 2023-11-16T13:28:35.658 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : udica-0.2.6-20.module_el8+712+4cd1bd69.noarch 10/13 2023-11-16T13:28:36.543 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : toolbox-0.0.99.4-6.module_el8+712+4cd1bd69.x86_64 11/13 2023-11-16T13:28:36.777 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : skopeo-2:1.13.3-1.module_el8+712+4cd1bd69.x86_64 12/13 2023-11-16T13:28:36.980 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 13/13 2023-11-16T13:28:37.398 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 13/13 2023-11-16T13:28:37.399 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : aardvark-dns-2:1.8.0-1.module_el8+712+4cd1bd69.x86 1/13 2023-11-16T13:28:37.399 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : buildah-1:1.32.0-1.module_el8+712+4cd1bd69.x86_64 2/13 2023-11-16T13:28:37.399 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 3/13 2023-11-16T13:28:37.399 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-11-16T13:28:37.399 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : crun-1.9.2-1.module_el8+712+4cd1bd69.x86_64 5/13 2023-11-16T13:28:37.399 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : netavark-2:1.8.0-2.module_el8+712+4cd1bd69.x86_64 6/13 2023-11-16T13:28:37.399 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-podman-4.7.0-1.module_el8+712+4cd1bd69.noa 7/13 2023-11-16T13:28:37.399 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2023-11-16T13:28:37.399 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2023-11-16T13:28:37.399 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : skopeo-2:1.13.3-1.module_el8+712+4cd1bd69.x86_64 10/13 2023-11-16T13:28:37.400 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : toolbox-0.0.99.4-6.module_el8+712+4cd1bd69.x86_64 11/13 2023-11-16T13:28:37.400 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : udica-0.2.6-20.module_el8+712+4cd1bd69.noarch 12/13 2023-11-16T13:28:38.216 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2023-11-16T13:28:38.217 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:28:38.217 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-11-16T13:28:38.217 INFO:teuthology.orchestra.run.smithi151.stdout: aardvark-dns-2:1.8.0-1.module_el8+712+4cd1bd69.x86_64 2023-11-16T13:28:38.217 INFO:teuthology.orchestra.run.smithi151.stdout: buildah-1:1.32.0-1.module_el8+712+4cd1bd69.x86_64 2023-11-16T13:28:38.217 INFO:teuthology.orchestra.run.smithi151.stdout: cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 2023-11-16T13:28:38.218 INFO:teuthology.orchestra.run.smithi151.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2023-11-16T13:28:38.218 INFO:teuthology.orchestra.run.smithi151.stdout: crun-1.9.2-1.module_el8+712+4cd1bd69.x86_64 2023-11-16T13:28:38.218 INFO:teuthology.orchestra.run.smithi151.stdout: netavark-2:1.8.0-2.module_el8+712+4cd1bd69.x86_64 2023-11-16T13:28:38.218 INFO:teuthology.orchestra.run.smithi151.stdout: python3-podman-4.7.0-1.module_el8+712+4cd1bd69.noarch 2023-11-16T13:28:38.218 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-11-16T13:28:38.219 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-11-16T13:28:38.219 INFO:teuthology.orchestra.run.smithi151.stdout: skopeo-2:1.13.3-1.module_el8+712+4cd1bd69.x86_64 2023-11-16T13:28:38.219 INFO:teuthology.orchestra.run.smithi151.stdout: toolbox-0.0.99.4-6.module_el8+712+4cd1bd69.x86_64 2023-11-16T13:28:38.220 INFO:teuthology.orchestra.run.smithi151.stdout: udica-0.2.6-20.module_el8+712+4cd1bd69.noarch 2023-11-16T13:28:38.220 INFO:teuthology.orchestra.run.smithi151.stdout: yajl-2.1.0-12.el8.x86_64 2023-11-16T13:28:38.220 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:28:38.220 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-11-16T13:28:38.497 DEBUG:teuthology.parallel:result is None 2023-11-16T13:28:38.976 INFO:teuthology.orchestra.run.smithi167.stdout:(13/13): buildah-1.32.0-1.module_el8+712+4cd1bd 628 kB/s | 9.6 MB 00:15 2023-11-16T13:28:38.976 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2023-11-16T13:28:38.976 INFO:teuthology.orchestra.run.smithi167.stdout:Total 1.7 MB/s | 27 MB 00:15 2023-11-16T13:28:39.249 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2023-11-16T13:28:39.270 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2023-11-16T13:28:39.270 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2023-11-16T13:28:39.529 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2023-11-16T13:28:39.532 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2023-11-16T13:28:40.244 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2023-11-16T13:28:40.448 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2023-11-16T13:28:40.610 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/13 2023-11-16T13:28:40.798 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/13 2023-11-16T13:28:41.079 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-11-16T13:28:41.642 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : aardvark-dns-2:1.8.0-1.module_el8+712+4cd1bd69.x86 5/13 2023-11-16T13:28:42.658 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : netavark-2:1.8.0-2.module_el8+712+4cd1bd69.x86_64 6/13 2023-11-16T13:28:42.880 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : buildah-1:1.32.0-1.module_el8+712+4cd1bd69.x86_64 7/13 2023-11-16T13:28:43.095 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : crun-1.9.2-1.module_el8+712+4cd1bd69.x86_64 8/13 2023-11-16T13:28:43.339 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-podman-4.7.0-1.module_el8+712+4cd1bd69.noa 9/13 2023-11-16T13:28:43.759 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : udica-0.2.6-20.module_el8+712+4cd1bd69.noarch 10/13 2023-11-16T13:28:44.677 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : toolbox-0.0.99.4-6.module_el8+712+4cd1bd69.x86_64 11/13 2023-11-16T13:28:44.889 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : skopeo-2:1.13.3-1.module_el8+712+4cd1bd69.x86_64 12/13 2023-11-16T13:28:45.092 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 13/13 2023-11-16T13:28:45.522 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 13/13 2023-11-16T13:28:45.522 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : aardvark-dns-2:1.8.0-1.module_el8+712+4cd1bd69.x86 1/13 2023-11-16T13:28:45.522 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : buildah-1:1.32.0-1.module_el8+712+4cd1bd69.x86_64 2/13 2023-11-16T13:28:45.523 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 3/13 2023-11-16T13:28:45.523 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-11-16T13:28:45.523 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : crun-1.9.2-1.module_el8+712+4cd1bd69.x86_64 5/13 2023-11-16T13:28:45.523 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : netavark-2:1.8.0-2.module_el8+712+4cd1bd69.x86_64 6/13 2023-11-16T13:28:45.523 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-podman-4.7.0-1.module_el8+712+4cd1bd69.noa 7/13 2023-11-16T13:28:45.523 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2023-11-16T13:28:45.523 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2023-11-16T13:28:45.523 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : skopeo-2:1.13.3-1.module_el8+712+4cd1bd69.x86_64 10/13 2023-11-16T13:28:45.523 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : toolbox-0.0.99.4-6.module_el8+712+4cd1bd69.x86_64 11/13 2023-11-16T13:28:45.523 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : udica-0.2.6-20.module_el8+712+4cd1bd69.noarch 12/13 2023-11-16T13:28:46.363 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2023-11-16T13:28:46.363 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:28:46.363 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2023-11-16T13:28:46.363 INFO:teuthology.orchestra.run.smithi167.stdout: aardvark-dns-2:1.8.0-1.module_el8+712+4cd1bd69.x86_64 2023-11-16T13:28:46.363 INFO:teuthology.orchestra.run.smithi167.stdout: buildah-1:1.32.0-1.module_el8+712+4cd1bd69.x86_64 2023-11-16T13:28:46.363 INFO:teuthology.orchestra.run.smithi167.stdout: cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 2023-11-16T13:28:46.364 INFO:teuthology.orchestra.run.smithi167.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2023-11-16T13:28:46.364 INFO:teuthology.orchestra.run.smithi167.stdout: crun-1.9.2-1.module_el8+712+4cd1bd69.x86_64 2023-11-16T13:28:46.364 INFO:teuthology.orchestra.run.smithi167.stdout: netavark-2:1.8.0-2.module_el8+712+4cd1bd69.x86_64 2023-11-16T13:28:46.364 INFO:teuthology.orchestra.run.smithi167.stdout: python3-podman-4.7.0-1.module_el8+712+4cd1bd69.noarch 2023-11-16T13:28:46.364 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-11-16T13:28:46.364 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-11-16T13:28:46.364 INFO:teuthology.orchestra.run.smithi167.stdout: skopeo-2:1.13.3-1.module_el8+712+4cd1bd69.x86_64 2023-11-16T13:28:46.364 INFO:teuthology.orchestra.run.smithi167.stdout: toolbox-0.0.99.4-6.module_el8+712+4cd1bd69.x86_64 2023-11-16T13:28:46.364 INFO:teuthology.orchestra.run.smithi167.stdout: udica-0.2.6-20.module_el8+712+4cd1bd69.noarch 2023-11-16T13:28:46.364 INFO:teuthology.orchestra.run.smithi167.stdout: yajl-2.1.0-12.el8.x86_64 2023-11-16T13:28:46.364 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:28:46.364 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2023-11-16T13:28:46.671 DEBUG:teuthology.parallel:result is None 2023-11-16T13:28:46.671 INFO:teuthology.run_tasks:Running task cephadm... 2023-11-16T13:28:46.744 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v16.2.5', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'docker.io/ceph/ceph:v16.2.5', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'ae6ce92c0812ba06c9c623ce72df4f3fc59080b4'} 2023-11-16T13:28:46.745 INFO:tasks.cephadm:Cluster image is docker.io/ceph/ceph:v16.2.5 2023-11-16T13:28:46.745 INFO:tasks.cephadm:Cluster fsid is 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:28:46.745 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-11-16T13:28:46.745 INFO:tasks.cephadm:No mon roles; fabricating mons 2023-11-16T13:28:46.745 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi151': '172.21.15.151', 'mon.smithi167': '172.21.15.167'} 2023-11-16T13:28:46.745 INFO:tasks.cephadm:Normalizing hostnames... 2023-11-16T13:28:46.746 DEBUG:teuthology.orchestra.run.smithi151:> sudo hostname $(hostname -s) 2023-11-16T13:28:46.780 DEBUG:teuthology.orchestra.run.smithi167:> sudo hostname $(hostname -s) 2023-11-16T13:28:46.807 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v16.2.5)... 2023-11-16T13:28:46.808 DEBUG:teuthology.orchestra.run.smithi151:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v16.2.5/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-11-16T13:28:47.065 INFO:teuthology.orchestra.run.smithi151.stdout:-rw-rw-r--. 1 ubuntu ubuntu 298325 Nov 16 13:28 /home/ubuntu/cephtest/cephadm 2023-11-16T13:28:47.066 DEBUG:teuthology.orchestra.run.smithi167:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v16.2.5/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-11-16T13:28:47.215 INFO:teuthology.orchestra.run.smithi167.stdout:-rw-rw-r--. 1 ubuntu ubuntu 298325 Nov 16 13:28 /home/ubuntu/cephtest/cephadm 2023-11-16T13:28:47.215 DEBUG:teuthology.orchestra.run.smithi151:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-11-16T13:28:47.238 DEBUG:teuthology.orchestra.run.smithi167:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-11-16T13:28:47.266 INFO:tasks.cephadm:Pulling image docker.io/ceph/ceph:v16.2.5 on all hosts... 2023-11-16T13:28:47.266 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 pull 2023-11-16T13:28:47.280 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 pull 2023-11-16T13:28:48.900 INFO:teuthology.orchestra.run.smithi151.stderr:Pulling container image docker.io/ceph/ceph:v16.2.5... 2023-11-16T13:28:48.960 INFO:teuthology.orchestra.run.smithi167.stderr:Pulling container image docker.io/ceph/ceph:v16.2.5... 2023-11-16T13:29:19.992 INFO:teuthology.orchestra.run.smithi167.stdout:{ 2023-11-16T13:29:19.992 INFO:teuthology.orchestra.run.smithi167.stdout: "ceph_version": "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)", 2023-11-16T13:29:19.992 INFO:teuthology.orchestra.run.smithi167.stdout: "image_id": "6933c2a0b7ddc222e77dc8d9dc471a0c639a1c0bded5077be53ad3a9557b4355", 2023-11-16T13:29:19.993 INFO:teuthology.orchestra.run.smithi167.stdout: "repo_digests": [ 2023-11-16T13:29:19.993 INFO:teuthology.orchestra.run.smithi167.stdout: "docker.io/ceph/ceph@sha256:829ebf54704f2d827de00913b171e5da741aad9b53c1f35ad59251524790eceb", 2023-11-16T13:29:19.993 INFO:teuthology.orchestra.run.smithi167.stdout: "docker.io/ceph/ceph@sha256:eb5600add17bccbe73ff1ee92faae1fce95007dcc51123f8c8a6258a090d2b03" 2023-11-16T13:29:19.993 INFO:teuthology.orchestra.run.smithi167.stdout: ] 2023-11-16T13:29:19.993 INFO:teuthology.orchestra.run.smithi167.stdout:} 2023-11-16T13:29:20.207 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:29:20.208 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph_version": "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)", 2023-11-16T13:29:20.208 INFO:teuthology.orchestra.run.smithi151.stdout: "image_id": "6933c2a0b7ddc222e77dc8d9dc471a0c639a1c0bded5077be53ad3a9557b4355", 2023-11-16T13:29:20.208 INFO:teuthology.orchestra.run.smithi151.stdout: "repo_digests": [ 2023-11-16T13:29:20.208 INFO:teuthology.orchestra.run.smithi151.stdout: "docker.io/ceph/ceph@sha256:829ebf54704f2d827de00913b171e5da741aad9b53c1f35ad59251524790eceb", 2023-11-16T13:29:20.208 INFO:teuthology.orchestra.run.smithi151.stdout: "docker.io/ceph/ceph@sha256:eb5600add17bccbe73ff1ee92faae1fce95007dcc51123f8c8a6258a090d2b03" 2023-11-16T13:29:20.208 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-11-16T13:29:20.208 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:29:20.226 DEBUG:teuthology.orchestra.run.smithi151:> sudo mkdir -p /etc/ceph 2023-11-16T13:29:20.256 DEBUG:teuthology.orchestra.run.smithi167:> sudo mkdir -p /etc/ceph 2023-11-16T13:29:20.286 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod 777 /etc/ceph 2023-11-16T13:29:20.324 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod 777 /etc/ceph 2023-11-16T13:29:20.353 INFO:tasks.cephadm:Writing seed config... 2023-11-16T13:29:20.354 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-11-16T13:29:20.354 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-11-16T13:29:20.354 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-11-16T13:29:20.354 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-11-16T13:29:20.354 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-11-16T13:29:20.354 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-11-16T13:29:20.354 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-11-16T13:29:20.355 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-11-16T13:29:20.355 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-11-16T13:29:20.355 DEBUG:teuthology.orchestra.run.smithi151:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-11-16T13:29:20.381 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 11fdb452-8484-11ee-8dbe-212e2dc638e7 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-11-16T13:29:20.382 DEBUG:teuthology.orchestra.run.smithi151:mon.smithi151> sudo journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi151.service 2023-11-16T13:29:20.423 INFO:tasks.cephadm:Bootstrapping... 2023-11-16T13:29:20.424 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 -v bootstrap --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.151 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-11-16T13:29:20.450 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:-- Logs begin at Thu 2023-11-16 13:20:07 UTC. -- 2023-11-16T13:29:20.617 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman version --format {{.Client.Version}} 2023-11-16T13:29:20.638 INFO:teuthology.orchestra.run.smithi151.stderr:/bin/podman: 4.6.1 2023-11-16T13:29:20.649 INFO:teuthology.orchestra.run.smithi151.stderr:Verifying podman|docker is present... 2023-11-16T13:29:20.649 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman version --format {{.Client.Version}} 2023-11-16T13:29:20.669 INFO:teuthology.orchestra.run.smithi151.stderr:/bin/podman: 4.6.1 2023-11-16T13:29:20.682 INFO:teuthology.orchestra.run.smithi151.stderr:Verifying lvm2 is present... 2023-11-16T13:29:20.682 INFO:teuthology.orchestra.run.smithi151.stderr:Verifying time synchronization is in place... 2023-11-16T13:29:20.682 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-enabled chrony.service 2023-11-16T13:29:20.688 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-11-16T13:29:20.688 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-active chrony.service 2023-11-16T13:29:20.693 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: inactive 2023-11-16T13:29:20.694 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-enabled chronyd.service 2023-11-16T13:29:20.699 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: enabled 2023-11-16T13:29:20.700 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-active chronyd.service 2023-11-16T13:29:20.706 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: active 2023-11-16T13:29:20.706 INFO:teuthology.orchestra.run.smithi151.stderr:Unit chronyd.service is enabled and running 2023-11-16T13:29:20.706 INFO:teuthology.orchestra.run.smithi151.stderr:Repeating the final host check... 2023-11-16T13:29:20.706 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman version --format {{.Client.Version}} 2023-11-16T13:29:20.724 INFO:teuthology.orchestra.run.smithi151.stderr:/bin/podman: 4.6.1 2023-11-16T13:29:20.734 INFO:teuthology.orchestra.run.smithi151.stderr:podman|docker (/bin/podman) is present 2023-11-16T13:29:20.734 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl is present 2023-11-16T13:29:20.734 INFO:teuthology.orchestra.run.smithi151.stderr:lvcreate is present 2023-11-16T13:29:20.734 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-enabled chrony.service 2023-11-16T13:29:20.739 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-11-16T13:29:20.740 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-active chrony.service 2023-11-16T13:29:20.745 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: inactive 2023-11-16T13:29:20.745 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-enabled chronyd.service 2023-11-16T13:29:20.751 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: enabled 2023-11-16T13:29:20.751 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-active chronyd.service 2023-11-16T13:29:20.756 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: active 2023-11-16T13:29:20.757 INFO:teuthology.orchestra.run.smithi151.stderr:Unit chronyd.service is enabled and running 2023-11-16T13:29:20.757 INFO:teuthology.orchestra.run.smithi151.stderr:Host looks OK 2023-11-16T13:29:20.757 INFO:teuthology.orchestra.run.smithi151.stderr:Cluster fsid: 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:29:20.757 INFO:teuthology.orchestra.run.smithi151.stderr:Acquiring lock 139848003272832 on /run/cephadm/11fdb452-8484-11ee-8dbe-212e2dc638e7.lock 2023-11-16T13:29:20.758 INFO:teuthology.orchestra.run.smithi151.stderr:Lock 139848003272832 acquired on /run/cephadm/11fdb452-8484-11ee-8dbe-212e2dc638e7.lock 2023-11-16T13:29:20.758 INFO:teuthology.orchestra.run.smithi151.stderr:Verifying IP 172.21.15.151 port 3300 ... 2023-11-16T13:29:20.758 INFO:teuthology.orchestra.run.smithi151.stderr:Verifying IP 172.21.15.151 port 6789 ... 2023-11-16T13:29:20.758 INFO:teuthology.orchestra.run.smithi151.stderr:Base mon IP is 172.21.15.151, final addrv is [v2:172.21.15.151:3300,v1:172.21.15.151:6789] 2023-11-16T13:29:20.758 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /sbin/ip route ls 2023-11-16T13:29:20.761 INFO:teuthology.orchestra.run.smithi151.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.151 metric 100 2023-11-16T13:29:20.761 INFO:teuthology.orchestra.run.smithi151.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.151 metric 100 2023-11-16T13:29:20.761 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /sbin/ip -6 route ls 2023-11-16T13:29:20.763 INFO:teuthology.orchestra.run.smithi151.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-11-16T13:29:20.763 INFO:teuthology.orchestra.run.smithi151.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2023-11-16T13:29:20.764 INFO:teuthology.orchestra.run.smithi151.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1797sec hoplimit 64 pref medium 2023-11-16T13:29:20.764 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /sbin/ip -6 addr ls 2023-11-16T13:29:20.766 INFO:teuthology.orchestra.run.smithi151.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-11-16T13:29:20.766 INFO:teuthology.orchestra.run.smithi151.stderr:/sbin/ip: inet6 ::1/128 scope host 2023-11-16T13:29:20.767 INFO:teuthology.orchestra.run.smithi151.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-11-16T13:29:20.767 INFO:teuthology.orchestra.run.smithi151.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-11-16T13:29:20.767 INFO:teuthology.orchestra.run.smithi151.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe88:6f63/64 scope link 2023-11-16T13:29:20.767 INFO:teuthology.orchestra.run.smithi151.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-11-16T13:29:20.767 INFO:teuthology.orchestra.run.smithi151.stderr:Mon IP `172.21.15.151` is in CIDR network `172.21.0.0/20` 2023-11-16T13:29:20.767 INFO:teuthology.orchestra.run.smithi151.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-11-16T13:29:20.768 INFO:teuthology.orchestra.run.smithi151.stderr:Pulling container image docker.io/ceph/ceph:v16.2.5... 2023-11-16T13:29:20.768 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman pull docker.io/ceph/ceph:v16.2.5 2023-11-16T13:29:20.848 INFO:teuthology.orchestra.run.smithi151.stderr:/bin/podman: Trying to pull docker.io/ceph/ceph:v16.2.5... 2023-11-16T13:29:27.299 INFO:teuthology.orchestra.run.smithi151.stderr:/bin/podman: Getting image source signatures 2023-11-16T13:29:27.299 INFO:teuthology.orchestra.run.smithi151.stderr:/bin/podman: Copying blob sha256:7a0437f04f83f084b7ed68ad9c4a4947e12fc4e1b006b38129bac89114ec3621 2023-11-16T13:29:27.299 INFO:teuthology.orchestra.run.smithi151.stderr:/bin/podman: Copying blob sha256:77890c5669f6b6c6924b8fca572f4f533c69239091810f2d2495b0df611f7263 2023-11-16T13:29:27.299 INFO:teuthology.orchestra.run.smithi151.stderr:/bin/podman: Copying config sha256:6933c2a0b7ddc222e77dc8d9dc471a0c639a1c0bded5077be53ad3a9557b4355 2023-11-16T13:29:27.360 INFO:teuthology.orchestra.run.smithi151.stderr:/bin/podman: Writing manifest to image destination 2023-11-16T13:29:27.471 INFO:teuthology.orchestra.run.smithi151.stderr:/bin/podman: 6933c2a0b7ddc222e77dc8d9dc471a0c639a1c0bded5077be53ad3a9557b4355 2023-11-16T13:29:27.482 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.5 --version 2023-11-16T13:29:27.820 INFO:teuthology.orchestra.run.smithi151.stderr:ceph: ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable) 2023-11-16T13:29:28.157 INFO:teuthology.orchestra.run.smithi151.stderr:Ceph version: ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable) 2023-11-16T13:29:28.158 INFO:teuthology.orchestra.run.smithi151.stderr:Extracting ceph user uid/gid from container image... 2023-11-16T13:29:28.158 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.5 -c %u %g /var/lib/ceph 2023-11-16T13:29:28.433 INFO:teuthology.orchestra.run.smithi151.stderr:stat: 167 167 2023-11-16T13:29:28.742 INFO:teuthology.orchestra.run.smithi151.stderr:Creating initial keys... 2023-11-16T13:29:28.742 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.5 --gen-print-key 2023-11-16T13:29:29.016 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-authtool: AQA4GVZl/L43OhAAgecYUBH8NuWaBHVyo1VcZw== 2023-11-16T13:29:29.332 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.5 --gen-print-key 2023-11-16T13:29:29.599 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-authtool: AQA5GVZlQ0msIhAAjUyX+INWJGDZ2R6juPoWkQ== 2023-11-16T13:29:29.928 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.5 --gen-print-key 2023-11-16T13:29:30.549 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-authtool: AQA6GVZlccqvHxAAfJn4mOfUe15ChedVgNNXqg== 2023-11-16T13:29:30.874 INFO:teuthology.orchestra.run.smithi151.stderr:Creating initial monmap... 2023-11-16T13:29:30.874 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/monmaptool --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /tmp/ceph-tmp0j9khcd4:/tmp/monmap:z docker.io/ceph/ceph:v16.2.5 --create --clobber --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 --addv smithi151 [v2:172.21.15.151:3300,v1:172.21.15.151:6789] /tmp/monmap 2023-11-16T13:29:31.124 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-11-16T13:29:31.124 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:29:31.124 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-11-16T13:29:31.419 INFO:teuthology.orchestra.run.smithi151.stderr:monmaptool for smithi151 [v2:172.21.15.151:3300,v1:172.21.15.151:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-11-16T13:29:31.419 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/monmaptool: set fsid to 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:29:31.419 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-11-16T13:29:31.419 INFO:teuthology.orchestra.run.smithi151.stderr: 2023-11-16T13:29:31.420 INFO:teuthology.orchestra.run.smithi151.stderr:Creating mon... 2023-11-16T13:29:31.421 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph-mon --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151:/var/lib/ceph/mon/ceph-smithi151:z -v /tmp/ceph-tmpwhuemznw:/tmp/keyring:z -v /tmp/ceph-tmp0j9khcd4:/tmp/monmap:z docker.io/ceph/ceph:v16.2.5 --mkfs -i smithi151 --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -c /dev/null --monmap /tmp/monmap --keyring /tmp/keyring --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true 2023-11-16T13:29:31.734 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.699+0000 7fecd9bd6700 0 set uid:gid to 167:167 (ceph:ceph) 2023-11-16T13:29:31.735 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.699+0000 7fecd9bd6700 1 imported monmap: 2023-11-16T13:29:31.735 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: epoch 0 2023-11-16T13:29:31.735 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:29:31.735 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: last_changed 2023-11-16T13:29:31.114936+0000 2023-11-16T13:29:31.735 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: created 2023-11-16T13:29:31.114936+0000 2023-11-16T13:29:31.735 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-11-16T13:29:31.736 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-11-16T13:29:31.736 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.smithi151 2023-11-16T13:29:31.736 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.736 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.699+0000 7fecd9bd6700 0 /usr/bin/ceph-mon: set fsid to 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:29:31.736 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: RocksDB version: 6.8.1 2023-11-16T13:29:31.736 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.736 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-11-16T13:29:31.737 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Compile date Jul 8 2021 2023-11-16T13:29:31.737 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: DB SUMMARY 2023-11-16T13:29:31.737 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.737 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi151/store.db dir, Total Num: 0, files: 2023-11-16T13:29:31.737 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.737 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi151/store.db: 2023-11-16T13:29:31.737 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.737 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.error_if_exists: 0 2023-11-16T13:29:31.738 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.create_if_missing: 1 2023-11-16T13:29:31.738 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.paranoid_checks: 1 2023-11-16T13:29:31.738 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.env: 0x561171a931c0 2023-11-16T13:29:31.738 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.fs: Posix File System 2023-11-16T13:29:31.738 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.info_log: 0x56117330d000 2023-11-16T13:29:31.738 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.max_file_opening_threads: 16 2023-11-16T13:29:31.738 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.statistics: (nil) 2023-11-16T13:29:31.739 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.use_fsync: 0 2023-11-16T13:29:31.739 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.max_log_file_size: 0 2023-11-16T13:29:31.739 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-11-16T13:29:31.739 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.log_file_time_to_roll: 0 2023-11-16T13:29:31.739 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.keep_log_file_num: 1000 2023-11-16T13:29:31.739 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.recycle_log_file_num: 0 2023-11-16T13:29:31.739 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.allow_fallocate: 1 2023-11-16T13:29:31.739 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.allow_mmap_reads: 0 2023-11-16T13:29:31.740 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.allow_mmap_writes: 0 2023-11-16T13:29:31.740 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.use_direct_reads: 0 2023-11-16T13:29:31.740 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-11-16T13:29:31.740 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.create_missing_column_families: 0 2023-11-16T13:29:31.740 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.db_log_dir: 2023-11-16T13:29:31.740 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi151/store.db 2023-11-16T13:29:31.740 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.table_cache_numshardbits: 6 2023-11-16T13:29:31.740 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.max_subcompactions: 1 2023-11-16T13:29:31.740 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.max_background_flushes: -1 2023-11-16T13:29:31.740 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-11-16T13:29:31.741 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-11-16T13:29:31.741 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-11-16T13:29:31.741 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-11-16T13:29:31.741 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-11-16T13:29:31.741 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.advise_random_on_open: 1 2023-11-16T13:29:31.741 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.db_write_buffer_size: 0 2023-11-16T13:29:31.741 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.write_buffer_manager: 0x561173315530 2023-11-16T13:29:31.741 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-11-16T13:29:31.741 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-11-16T13:29:31.741 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-11-16T13:29:31.742 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.use_adaptive_mutex: 0 2023-11-16T13:29:31.742 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.rate_limiter: (nil) 2023-11-16T13:29:31.742 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-11-16T13:29:31.742 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.wal_recovery_mode: 2 2023-11-16T13:29:31.742 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.enable_thread_tracking: 0 2023-11-16T13:29:31.742 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.enable_pipelined_write: 0 2023-11-16T13:29:31.742 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.unordered_write: 0 2023-11-16T13:29:31.742 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-11-16T13:29:31.742 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-11-16T13:29:31.742 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-11-16T13:29:31.743 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-11-16T13:29:31.743 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.row_cache: None 2023-11-16T13:29:31.743 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.wal_filter: None 2023-11-16T13:29:31.743 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-11-16T13:29:31.743 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.allow_ingest_behind: 0 2023-11-16T13:29:31.743 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.preserve_deletes: 0 2023-11-16T13:29:31.743 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.two_write_queues: 0 2023-11-16T13:29:31.743 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.manual_wal_flush: 0 2023-11-16T13:29:31.743 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.atomic_flush: 0 2023-11-16T13:29:31.743 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-11-16T13:29:31.744 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.persist_stats_to_disk: 0 2023-11-16T13:29:31.744 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-11-16T13:29:31.744 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.log_readahead_size: 0 2023-11-16T13:29:31.744 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-11-16T13:29:31.744 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.max_background_jobs: 2 2023-11-16T13:29:31.744 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.max_background_compactions: -1 2023-11-16T13:29:31.744 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-11-16T13:29:31.744 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-11-16T13:29:31.744 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.delayed_write_rate : 16777216 2023-11-16T13:29:31.745 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.max_total_wal_size: 0 2023-11-16T13:29:31.745 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-11-16T13:29:31.745 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.stats_dump_period_sec: 600 2023-11-16T13:29:31.745 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.stats_persist_period_sec: 600 2023-11-16T13:29:31.745 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-11-16T13:29:31.745 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.max_open_files: -1 2023-11-16T13:29:31.745 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.bytes_per_sync: 0 2023-11-16T13:29:31.745 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-11-16T13:29:31.745 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-11-16T13:29:31.745 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Options.compaction_readahead_size: 0 2023-11-16T13:29:31.745 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Compression algorithms supported: 2023-11-16T13:29:31.746 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-11-16T13:29:31.746 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: kZSTD supported: 0 2023-11-16T13:29:31.746 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: kXpressCompression supported: 0 2023-11-16T13:29:31.746 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: kLZ4HCCompression supported: 1 2023-11-16T13:29:31.746 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: kLZ4Compression supported: 1 2023-11-16T13:29:31.746 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: kBZip2Compression supported: 0 2023-11-16T13:29:31.746 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: kZlibCompression supported: 1 2023-11-16T13:29:31.747 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: kSnappyCompression supported: 1 2023-11-16T13:29:31.747 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.700+0000 7fecd9bd6700 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-11-16T13:29:31.747 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.701+0000 7fecd9bd6700 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-11-16T13:29:31.747 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.747 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi151/store.db/MANIFEST-000001 2023-11-16T13:29:31.747 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.747 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-11-16T13:29:31.748 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.748 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-11-16T13:29:31.748 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.merge_operator: 2023-11-16T13:29:31.748 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.compaction_filter: None 2023-11-16T13:29:31.748 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.compaction_filter_factory: None 2023-11-16T13:29:31.748 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-11-16T13:29:31.748 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.table_factory: BlockBasedTable 2023-11-16T13:29:31.748 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561173226160) 2023-11-16T13:29:31.749 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-11-16T13:29:31.749 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-11-16T13:29:31.749 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-11-16T13:29:31.749 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-11-16T13:29:31.749 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: index_type: 0 2023-11-16T13:29:31.749 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-11-16T13:29:31.750 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-11-16T13:29:31.750 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-11-16T13:29:31.750 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-11-16T13:29:31.750 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: checksum: 1 2023-11-16T13:29:31.750 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-11-16T13:29:31.750 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: block_cache: 0x56117325cd10 2023-11-16T13:29:31.750 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-11-16T13:29:31.751 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-11-16T13:29:31.751 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-11-16T13:29:31.751 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-11-16T13:29:31.751 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-11-16T13:29:31.751 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-11-16T13:29:31.751 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-11-16T13:29:31.751 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-11-16T13:29:31.751 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-11-16T13:29:31.752 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-11-16T13:29:31.752 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-11-16T13:29:31.752 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-11-16T13:29:31.752 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-11-16T13:29:31.752 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-11-16T13:29:31.752 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-11-16T13:29:31.752 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-11-16T13:29:31.752 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-11-16T13:29:31.753 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-11-16T13:29:31.753 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-11-16T13:29:31.753 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: format_version: 2 2023-11-16T13:29:31.753 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-11-16T13:29:31.753 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: block_align: 0 2023-11-16T13:29:31.753 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.753 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.write_buffer_size: 33554432 2023-11-16T13:29:31.753 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.max_write_buffer_number: 2 2023-11-16T13:29:31.753 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.compression: NoCompression 2023-11-16T13:29:31.754 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.bottommost_compression: Disabled 2023-11-16T13:29:31.754 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.prefix_extractor: nullptr 2023-11-16T13:29:31.754 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-11-16T13:29:31.754 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.num_levels: 7 2023-11-16T13:29:31.754 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-11-16T13:29:31.754 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-11-16T13:29:31.755 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-11-16T13:29:31.755 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-11-16T13:29:31.755 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-11-16T13:29:31.755 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.706+0000 7fecd9bd6700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-11-16T13:29:31.755 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-11-16T13:29:31.755 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-11-16T13:29:31.755 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-11-16T13:29:31.755 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compression_opts.window_bits: -14 2023-11-16T13:29:31.756 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compression_opts.level: 32767 2023-11-16T13:29:31.756 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compression_opts.strategy: 0 2023-11-16T13:29:31.756 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-11-16T13:29:31.756 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-11-16T13:29:31.756 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compression_opts.enabled: false 2023-11-16T13:29:31.756 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-11-16T13:29:31.756 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-11-16T13:29:31.757 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-11-16T13:29:31.757 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.target_file_size_base: 67108864 2023-11-16T13:29:31.757 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.target_file_size_multiplier: 1 2023-11-16T13:29:31.757 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-11-16T13:29:31.757 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-11-16T13:29:31.757 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-11-16T13:29:31.757 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-11-16T13:29:31.758 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-11-16T13:29:31.758 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-11-16T13:29:31.758 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-11-16T13:29:31.758 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-11-16T13:29:31.758 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-11-16T13:29:31.758 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-11-16T13:29:31.758 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-11-16T13:29:31.759 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-11-16T13:29:31.759 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.arena_block_size: 4194304 2023-11-16T13:29:31.759 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-11-16T13:29:31.759 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-11-16T13:29:31.759 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-11-16T13:29:31.759 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.disable_auto_compactions: 0 2023-11-16T13:29:31.759 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-11-16T13:29:31.759 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-11-16T13:29:31.760 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-11-16T13:29:31.760 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-11-16T13:29:31.760 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-11-16T13:29:31.760 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-11-16T13:29:31.760 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-11-16T13:29:31.760 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-11-16T13:29:31.760 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-11-16T13:29:31.760 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-11-16T13:29:31.760 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.table_properties_collectors: 2023-11-16T13:29:31.760 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.inplace_update_support: 0 2023-11-16T13:29:31.761 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-11-16T13:29:31.761 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-11-16T13:29:31.761 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-11-16T13:29:31.761 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.memtable_huge_page_size: 0 2023-11-16T13:29:31.761 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.bloom_locality: 0 2023-11-16T13:29:31.761 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.max_successive_merges: 0 2023-11-16T13:29:31.761 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-11-16T13:29:31.761 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.paranoid_file_checks: 0 2023-11-16T13:29:31.761 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.force_consistency_checks: 0 2023-11-16T13:29:31.761 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.report_bg_io_stats: 0 2023-11-16T13:29:31.761 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.ttl: 2592000 2023-11-16T13:29:31.762 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-11-16T13:29:31.762 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi151/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-11-16T13:29:31.762 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.762 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.707+0000 7fecd9bd6700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-11-16T13:29:31.762 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.762 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.708+0000 7fecd9bd6700 4 rocksdb: DB pointer 0x561173321800 2023-11-16T13:29:31.763 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.708+0000 7fecc2a5f700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-11-16T13:29:31.765 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.708+0000 7fecc2a5f700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-11-16T13:29:31.765 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-11-16T13:29:31.765 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-11-16T13:29:31.765 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-11-16T13:29:31.765 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-11-16T13:29:31.765 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-16T13:29:31.765 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-11-16T13:29:31.766 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-11-16T13:29:31.766 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-16T13:29:31.766 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.766 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-11-16T13:29:31.766 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:29:31.766 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:29:31.766 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-16T13:29:31.766 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-16T13:29:31.766 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.766 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-11-16T13:29:31.766 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:29:31.767 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:29:31.767 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-11-16T13:29:31.767 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-11-16T13:29:31.767 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-16T13:29:31.767 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-11-16T13:29:31.767 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-11-16T13:29:31.767 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-11-16T13:29:31.767 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:29:31.767 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:29:31.767 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-16T13:29:31.768 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.768 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-11-16T13:29:31.768 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.768 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-11-16T13:29:31.768 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:29:31.768 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:29:31.768 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-16T13:29:31.768 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-16T13:29:31.768 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.769 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-11-16T13:29:31.769 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:29:31.769 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:29:31.769 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-11-16T13:29:31.770 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-11-16T13:29:31.770 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-16T13:29:31.770 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-11-16T13:29:31.770 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-11-16T13:29:31.770 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-11-16T13:29:31.770 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:29:31.770 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:29:31.770 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-16T13:29:31.770 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.771 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-11-16T13:29:31.771 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: 2023-11-16T13:29:31.771 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.709+0000 7fecd9bd6700 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-11-16T13:29:31.771 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.709+0000 7fecd9bd6700 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-11-16T13:29:31.771 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph-mon: debug 2023-11-16T13:29:31.709+0000 7fecd9bd6700 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-smithi151 for mon.smithi151 2023-11-16T13:29:32.060 INFO:teuthology.orchestra.run.smithi151.stderr:create mon.smithi151 on 2023-11-16T13:29:32.060 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: install -d -m0770 -o 167 -g 167 /var/run/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:29:32.065 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl disable ceph.target 2023-11-16T13:29:32.153 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl enable ceph.target 2023-11-16T13:29:32.160 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-11-16T13:29:32.238 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl start ceph.target 2023-11-16T13:29:32.245 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl enable ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7.target 2023-11-16T13:29:32.252 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7.target → /etc/systemd/system/ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7.target. 2023-11-16T13:29:32.252 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7.target → /etc/systemd/system/ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7.target. 2023-11-16T13:29:32.328 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl start ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7.target 2023-11-16T13:29:32.336 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl daemon-reload 2023-11-16T13:29:32.414 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl stop ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi151 2023-11-16T13:29:32.438 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl reset-failed ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi151 2023-11-16T13:29:32.443 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: Failed to reset failed state of unit ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi151.service: Unit ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi151.service not loaded. 2023-11-16T13:29:32.444 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl enable ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi151 2023-11-16T13:29:32.450 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: Created symlink /etc/systemd/system/ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7.target.wants/ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi151.service → /etc/systemd/system/ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@.service. 2023-11-16T13:29:32.533 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl start ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi151 2023-11-16T13:29:32.844 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-enabled firewalld.service 2023-11-16T13:29:32.852 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: disabled 2023-11-16T13:29:32.852 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-active firewalld.service 2023-11-16T13:29:32.859 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: inactive 2023-11-16T13:29:32.860 INFO:teuthology.orchestra.run.smithi151.stderr:firewalld.service is not enabled 2023-11-16T13:29:32.860 INFO:teuthology.orchestra.run.smithi151.stderr:Not possible to enable service . firewalld.service is not available 2023-11-16T13:29:32.861 INFO:teuthology.orchestra.run.smithi151.stderr:Waiting for mon to start... 2023-11-16T13:29:32.861 INFO:teuthology.orchestra.run.smithi151.stderr:Waiting for mon... 2023-11-16T13:29:32.861 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151:/var/lib/ceph/mon/ceph-smithi151:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 status 2023-11-16T13:29:32.873 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:32 smithi151 bash[107141]: 0c060846c2968236aca2e4159e3a98ba118f3018e3c62d6a9dd964ce570be42d 2023-11-16T13:29:32.873 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:32 smithi151 systemd[1]: Started Ceph mon.smithi151 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:29:33.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107156]: cluster 2023-11-16T13:29:32.864506+0000 mon.smithi151 (mon.0) 0 : [INF] mkfs 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:29:33.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107156]: cluster 2023-11-16T13:29:32.851596+0000 mon.smithi151 (mon.0) 1 : cluster [INF] mon.smithi151 is new leader, mons smithi151 in quorum (ranks 0) 2023-11-16T13:29:33.350 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: cluster: 2023-11-16T13:29:33.350 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: id: 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:29:33.350 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-11-16T13:29:33.350 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 2023-11-16T13:29:33.350 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: services: 2023-11-16T13:29:33.350 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi151 (age 0.488249s) 2023-11-16T13:29:33.351 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: mgr: no daemons active 2023-11-16T13:29:33.351 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-11-16T13:29:33.351 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 2023-11-16T13:29:33.351 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: data: 2023-11-16T13:29:33.351 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-11-16T13:29:33.351 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-11-16T13:29:33.351 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-11-16T13:29:33.351 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: pgs: 2023-11-16T13:29:33.351 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 2023-11-16T13:29:33.685 INFO:teuthology.orchestra.run.smithi151.stderr:mon is available 2023-11-16T13:29:33.685 INFO:teuthology.orchestra.run.smithi151.stderr:Assimilating anything we can from ceph.conf... 2023-11-16T13:29:33.685 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z -v /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151:/var/lib/ceph/mon/ceph-smithi151:z docker.io/ceph/ceph:v16.2.5 config assimilate-conf -i /var/lib/ceph/mon/ceph-smithi151/config 2023-11-16T13:29:34.129 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 2023-11-16T13:29:34.130 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: [global] 2023-11-16T13:29:34.130 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: fsid = 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:29:34.130 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: log_to_journald = false 2023-11-16T13:29:34.130 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.151:3300,v1:172.21.15.151:6789] 2023-11-16T13:29:34.130 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-11-16T13:29:34.130 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-11-16T13:29:34.130 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-11-16T13:29:34.130 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-11-16T13:29:34.130 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 2023-11-16T13:29:34.130 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: [mgr] 2023-11-16T13:29:34.131 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-11-16T13:29:34.131 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 2023-11-16T13:29:34.131 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: [osd] 2023-11-16T13:29:34.131 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-11-16T13:29:34.131 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-11-16T13:29:34.155 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107156]: cluster 2023-11-16T13:29:32.859150+0000 mon.smithi151 (mon.0) 2 : cluster [INF] mon.smithi151 is new leader, mons smithi151 in quorum (ranks 0) 2023-11-16T13:29:34.156 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107156]: cluster 2023-11-16T13:29:32.861451+0000 mon.smithi151 (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {smithi151=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0]} 2023-11-16T13:29:34.156 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107156]: cluster 2023-11-16T13:29:32.868877+0000 mon.smithi151 (mon.0) 4 : cluster [DBG] fsmap 2023-11-16T13:29:34.156 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107156]: cluster 2023-11-16T13:29:32.874161+0000 mon.smithi151 (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-11-16T13:29:34.156 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107156]: cluster 2023-11-16T13:29:32.874732+0000 mon.smithi151 (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-11-16T13:29:34.156 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107156]: audit 2023-11-16T13:29:33.347366+0000 mon.smithi151 (mon.0) 7 : audit [DBG] from='client.? 172.21.15.151:0/3405903440' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-11-16T13:29:34.484 INFO:teuthology.orchestra.run.smithi151.stderr:Generating new minimal ceph.conf... 2023-11-16T13:29:34.485 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z -v /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151:/var/lib/ceph/mon/ceph-smithi151:z docker.io/ceph/ceph:v16.2.5 config generate-minimal-conf -o /var/lib/ceph/mon/ceph-smithi151/config 2023-11-16T13:29:35.161 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107156]: audit 2023-11-16T13:29:34.123892+0000 mon.smithi151 (mon.0) 8 : audit [INF] from='client.? 172.21.15.151:0/2833170223' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-11-16T13:29:35.161 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107156]: audit 2023-11-16T13:29:34.127311+0000 mon.smithi151 (mon.0) 9 : audit [INF] from='client.? 172.21.15.151:0/2833170223' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-11-16T13:29:35.234 INFO:teuthology.orchestra.run.smithi151.stderr:Restarting the monitor... 2023-11-16T13:29:35.235 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl restart ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi151 2023-11-16T13:29:35.450 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 systemd[1]: Stopping Ceph mon.smithi151 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:29:35.451 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107156]: debug 2023-11-16T13:29:35.314+0000 7fd478c5d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi151 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-11-16T13:29:35.451 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107156]: debug 2023-11-16T13:29:35.314+0000 7fd478c5d700 -1 mon.smithi151@0(leader) e1 *** Got Signal Terminated *** 2023-11-16T13:29:35.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 podman[107477]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151 2023-11-16T13:29:35.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 systemd[1]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi151.service: Succeeded. 2023-11-16T13:29:35.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 systemd[1]: Stopped Ceph mon.smithi151 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:29:35.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 systemd[1]: Starting Ceph mon.smithi151 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:29:35.835 INFO:teuthology.orchestra.run.smithi151.stderr:Setting mon public_network to 172.21.0.0/20 2023-11-16T13:29:35.835 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 config set mon public_network 172.21.0.0/20 2023-11-16T13:29:36.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.821+0000 7f8368b4e700 0 set uid:gid to 167:167 (ceph:ceph) 2023-11-16T13:29:36.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.821+0000 7f8368b4e700 0 ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable), process ceph-mon, pid 7 2023-11-16T13:29:36.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.821+0000 7f8368b4e700 0 pidfile_write: ignore empty --pid-file 2023-11-16T13:29:36.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 0 load: jerasure load: lrc load: isa 2023-11-16T13:29:36.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: RocksDB version: 6.8.1 2023-11-16T13:29:36.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-11-16T13:29:36.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Compile date Jul 8 2021 2023-11-16T13:29:36.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: DB SUMMARY 2023-11-16T13:29:36.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: CURRENT file: CURRENT 2023-11-16T13:29:36.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: IDENTITY file: IDENTITY 2023-11-16T13:29:36.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-11-16T13:29:36.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi151/store.db dir, Total Num: 1, files: 000004.sst 2023-11-16T13:29:36.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi151/store.db: 000006.log size: 80640 ; 2023-11-16T13:29:36.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.error_if_exists: 0 2023-11-16T13:29:36.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.create_if_missing: 0 2023-11-16T13:29:36.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.paranoid_checks: 1 2023-11-16T13:29:36.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.env: 0x55bfa3c771c0 2023-11-16T13:29:36.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.fs: Posix File System 2023-11-16T13:29:36.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.info_log: 0x55bfa4beda40 2023-11-16T13:29:36.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.max_file_opening_threads: 16 2023-11-16T13:29:36.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.statistics: (nil) 2023-11-16T13:29:36.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.use_fsync: 0 2023-11-16T13:29:36.238 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.max_log_file_size: 0 2023-11-16T13:29:36.238 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-11-16T13:29:36.238 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.log_file_time_to_roll: 0 2023-11-16T13:29:36.238 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.keep_log_file_num: 1000 2023-11-16T13:29:36.238 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.recycle_log_file_num: 0 2023-11-16T13:29:36.238 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.allow_fallocate: 1 2023-11-16T13:29:36.238 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.allow_mmap_reads: 0 2023-11-16T13:29:36.238 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.allow_mmap_writes: 0 2023-11-16T13:29:36.238 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.use_direct_reads: 0 2023-11-16T13:29:36.238 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-11-16T13:29:36.238 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.create_missing_column_families: 0 2023-11-16T13:29:36.239 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.db_log_dir: 2023-11-16T13:29:36.239 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi151/store.db 2023-11-16T13:29:36.239 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.table_cache_numshardbits: 6 2023-11-16T13:29:36.239 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.max_subcompactions: 1 2023-11-16T13:29:36.239 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.max_background_flushes: -1 2023-11-16T13:29:36.239 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-11-16T13:29:36.239 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-11-16T13:29:36.239 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-11-16T13:29:36.239 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-11-16T13:29:36.239 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-11-16T13:29:36.239 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.advise_random_on_open: 1 2023-11-16T13:29:36.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.db_write_buffer_size: 0 2023-11-16T13:29:36.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.write_buffer_manager: 0x55bfa4bf5fb0 2023-11-16T13:29:36.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-11-16T13:29:36.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-11-16T13:29:36.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-11-16T13:29:36.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.use_adaptive_mutex: 0 2023-11-16T13:29:36.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.rate_limiter: (nil) 2023-11-16T13:29:36.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-11-16T13:29:36.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.wal_recovery_mode: 2 2023-11-16T13:29:36.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.enable_thread_tracking: 0 2023-11-16T13:29:36.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.enable_pipelined_write: 0 2023-11-16T13:29:36.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.unordered_write: 0 2023-11-16T13:29:36.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-11-16T13:29:36.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-11-16T13:29:36.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-11-16T13:29:36.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-11-16T13:29:36.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.row_cache: None 2023-11-16T13:29:36.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.wal_filter: None 2023-11-16T13:29:36.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-11-16T13:29:36.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.allow_ingest_behind: 0 2023-11-16T13:29:36.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.preserve_deletes: 0 2023-11-16T13:29:36.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.two_write_queues: 0 2023-11-16T13:29:36.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.manual_wal_flush: 0 2023-11-16T13:29:36.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.atomic_flush: 0 2023-11-16T13:29:36.242 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-11-16T13:29:36.242 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.persist_stats_to_disk: 0 2023-11-16T13:29:36.242 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-11-16T13:29:36.242 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.log_readahead_size: 0 2023-11-16T13:29:36.242 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-11-16T13:29:36.242 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.max_background_jobs: 2 2023-11-16T13:29:36.242 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.max_background_compactions: -1 2023-11-16T13:29:36.242 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-11-16T13:29:36.242 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-11-16T13:29:36.242 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.delayed_write_rate : 16777216 2023-11-16T13:29:36.242 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.max_total_wal_size: 0 2023-11-16T13:29:36.243 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-11-16T13:29:36.243 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.stats_dump_period_sec: 600 2023-11-16T13:29:36.243 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.stats_persist_period_sec: 600 2023-11-16T13:29:36.243 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-11-16T13:29:36.243 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.max_open_files: -1 2023-11-16T13:29:36.243 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.bytes_per_sync: 0 2023-11-16T13:29:36.243 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-11-16T13:29:36.243 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-11-16T13:29:36.243 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.compaction_readahead_size: 0 2023-11-16T13:29:36.243 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Compression algorithms supported: 2023-11-16T13:29:36.243 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-11-16T13:29:36.243 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: kZSTD supported: 0 2023-11-16T13:29:36.244 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: kXpressCompression supported: 0 2023-11-16T13:29:36.244 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: kLZ4HCCompression supported: 1 2023-11-16T13:29:36.244 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: kLZ4Compression supported: 1 2023-11-16T13:29:36.244 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: kBZip2Compression supported: 0 2023-11-16T13:29:36.244 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: kZlibCompression supported: 1 2023-11-16T13:29:36.244 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: kSnappyCompression supported: 1 2023-11-16T13:29:36.244 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 bash[107548]: 3a504a936c5d9b82e63f6ae83035d23ddb9dd638a8a2fb03240ab524e5905b77 2023-11-16T13:29:36.244 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-11-16T13:29:36.244 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi151/store.db/MANIFEST-000005 2023-11-16T13:29:36.244 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.244 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-11-16T13:29:36.244 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.245 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-11-16T13:29:36.245 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.merge_operator: 2023-11-16T13:29:36.245 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.compaction_filter: None 2023-11-16T13:29:36.245 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.compaction_filter_factory: None 2023-11-16T13:29:36.245 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-11-16T13:29:36.245 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.826+0000 7f8368b4e700 4 rocksdb: Options.table_factory: BlockBasedTable 2023-11-16T13:29:36.245 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55bfa4b07cc0) 2023-11-16T13:29:36.245 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cache_index_and_filter_blocks: 1 2023-11-16T13:29:36.245 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cache_index_and_filter_blocks_with_high_priority: 0 2023-11-16T13:29:36.245 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-11-16T13:29:36.245 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: pin_top_level_index_and_filter: 1 2023-11-16T13:29:36.245 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: index_type: 0 2023-11-16T13:29:36.246 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: data_block_index_type: 0 2023-11-16T13:29:36.246 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: index_shortening: 1 2023-11-16T13:29:36.246 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: data_block_hash_table_util_ratio: 0.750000 2023-11-16T13:29:36.246 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: hash_index_allow_collision: 1 2023-11-16T13:29:36.246 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: checksum: 1 2023-11-16T13:29:36.246 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: no_block_cache: 0 2023-11-16T13:29:36.246 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: block_cache: 0x55bfa4b3cf10 2023-11-16T13:29:36.246 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: block_cache_name: BinnedLRUCache 2023-11-16T13:29:36.246 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: block_cache_options: 2023-11-16T13:29:36.246 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: capacity : 536870912 2023-11-16T13:29:36.246 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: num_shard_bits : 4 2023-11-16T13:29:36.247 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: strict_capacity_limit : 0 2023-11-16T13:29:36.247 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: high_pri_pool_ratio: 0.000 2023-11-16T13:29:36.247 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: block_cache_compressed: (nil) 2023-11-16T13:29:36.247 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: persistent_cache: (nil) 2023-11-16T13:29:36.247 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: block_size: 4096 2023-11-16T13:29:36.247 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: block_size_deviation: 10 2023-11-16T13:29:36.247 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: block_restart_interval: 16 2023-11-16T13:29:36.247 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: index_block_restart_interval: 1 2023-11-16T13:29:36.247 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: metadata_block_size: 4096 2023-11-16T13:29:36.247 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: partition_filters: 0 2023-11-16T13:29:36.247 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: use_delta_encoding: 1 2023-11-16T13:29:36.248 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: filter_policy: rocksdb.BuiltinBloomFilter 2023-11-16T13:29:36.248 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: whole_key_filtering: 1 2023-11-16T13:29:36.248 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: verify_compression: 0 2023-11-16T13:29:36.248 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: read_amp_bytes_per_bit: 0 2023-11-16T13:29:36.248 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: format_version: 2 2023-11-16T13:29:36.248 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: enable_index_compression: 1 2023-11-16T13:29:36.248 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: block_align: 0 2023-11-16T13:29:36.248 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.248 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.write_buffer_size: 33554432 2023-11-16T13:29:36.248 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_write_buffer_number: 2 2023-11-16T13:29:36.248 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compression: NoCompression 2023-11-16T13:29:36.248 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.bottommost_compression: Disabled 2023-11-16T13:29:36.249 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.prefix_extractor: nullptr 2023-11-16T13:29:36.249 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-11-16T13:29:36.249 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.num_levels: 7 2023-11-16T13:29:36.249 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-11-16T13:29:36.249 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-11-16T13:29:36.249 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-11-16T13:29:36.249 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-11-16T13:29:36.249 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-11-16T13:29:36.249 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-11-16T13:29:36.249 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-11-16T13:29:36.249 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-11-16T13:29:36.249 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-11-16T13:29:36.250 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compression_opts.window_bits: -14 2023-11-16T13:29:36.250 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compression_opts.level: 32767 2023-11-16T13:29:36.250 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compression_opts.strategy: 0 2023-11-16T13:29:36.250 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-11-16T13:29:36.250 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-11-16T13:29:36.250 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compression_opts.enabled: false 2023-11-16T13:29:36.250 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-11-16T13:29:36.250 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-11-16T13:29:36.250 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-11-16T13:29:36.250 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.target_file_size_base: 67108864 2023-11-16T13:29:36.250 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.target_file_size_multiplier: 1 2023-11-16T13:29:36.250 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-11-16T13:29:36.251 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-11-16T13:29:36.251 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-11-16T13:29:36.251 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-11-16T13:29:36.251 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-11-16T13:29:36.251 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-11-16T13:29:36.251 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-11-16T13:29:36.251 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-11-16T13:29:36.251 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-11-16T13:29:36.251 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-11-16T13:29:36.251 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-11-16T13:29:36.251 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-11-16T13:29:36.251 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.arena_block_size: 4194304 2023-11-16T13:29:36.252 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-11-16T13:29:36.252 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-11-16T13:29:36.252 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-11-16T13:29:36.252 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.disable_auto_compactions: 0 2023-11-16T13:29:36.252 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-11-16T13:29:36.252 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-11-16T13:29:36.252 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-11-16T13:29:36.252 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-11-16T13:29:36.252 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-11-16T13:29:36.252 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-11-16T13:29:36.252 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-11-16T13:29:36.252 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-11-16T13:29:36.253 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-11-16T13:29:36.253 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-11-16T13:29:36.253 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.table_properties_collectors: 2023-11-16T13:29:36.253 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.inplace_update_support: 0 2023-11-16T13:29:36.253 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-11-16T13:29:36.253 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-11-16T13:29:36.253 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-11-16T13:29:36.253 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.memtable_huge_page_size: 0 2023-11-16T13:29:36.253 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.bloom_locality: 0 2023-11-16T13:29:36.253 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.max_successive_merges: 0 2023-11-16T13:29:36.253 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-11-16T13:29:36.253 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.paranoid_file_checks: 0 2023-11-16T13:29:36.254 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.force_consistency_checks: 0 2023-11-16T13:29:36.254 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.report_bg_io_stats: 0 2023-11-16T13:29:36.254 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.ttl: 2592000 2023-11-16T13:29:36.254 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-11-16T13:29:36.254 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi151/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-11-16T13:29:36.254 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.254 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-11-16T13:29:36.254 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.254 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1700141375828638, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-11-16T13:29:36.254 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.827+0000 7f8368b4e700 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-11-16T13:29:36.254 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.828+0000 7f8368b4e700 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-11-16T13:29:36.255 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.828+0000 7f8368b4e700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1700141375829436, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 77535, "table_properties": {"data_size": 75830, "index_size": 310, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9398, "raw_average_key_size": 47, "raw_value_size": 70402, "raw_average_value_size": 357, "num_data_blocks": 10, "num_entries": 197, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1700141375, "oldest_key_time": 3, "file_creation_time": 0}} 2023-11-16T13:29:36.255 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.828+0000 7f8368b4e700 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-11-16T13:29:36.255 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.255 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.829+0000 7f8368b4e700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1700141375829842, "job": 1, "event": "recovery_finished"} 2023-11-16T13:29:36.255 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.829+0000 7f8368b4e700 4 rocksdb: DB pointer 0x55bfa4c01800 2023-11-16T13:29:36.255 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.829+0000 7f834f956700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-11-16T13:29:36.255 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.829+0000 7f834f956700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-11-16T13:29:36.255 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: ** DB Stats ** 2023-11-16T13:29:36.255 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Uptime(secs): 0.0 total, 0.0 interval 2023-11-16T13:29:36.256 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-11-16T13:29:36.256 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-11-16T13:29:36.256 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-16T13:29:36.256 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-11-16T13:29:36.257 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-11-16T13:29:36.257 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-16T13:29:36.257 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.257 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: ** Compaction Stats [default] ** 2023-11-16T13:29:36.257 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:29:36.257 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:29:36.257 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: L0 2/0 77.41 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.7 0.00 0.00 1 0.001 0 0 2023-11-16T13:29:36.257 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Sum 2/0 77.41 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.7 0.00 0.00 1 0.001 0 0 2023-11-16T13:29:36.257 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.7 0.00 0.00 1 0.001 0 0 2023-11-16T13:29:36.257 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.257 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: ** Compaction Stats [default] ** 2023-11-16T13:29:36.257 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:29:36.258 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:29:36.258 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 137.7 0.00 0.00 1 0.001 0 0 2023-11-16T13:29:36.258 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Uptime(secs): 0.0 total, 0.0 interval 2023-11-16T13:29:36.258 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Flush(GB): cumulative 0.000, interval 0.000 2023-11-16T13:29:36.258 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-16T13:29:36.258 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: AddFile(Total Files): cumulative 0, interval 0 2023-11-16T13:29:36.258 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: AddFile(L0 Files): cumulative 0, interval 0 2023-11-16T13:29:36.258 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: AddFile(Keys): cumulative 0, interval 0 2023-11-16T13:29:36.258 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Cumulative compaction: 0.00 GB write, 26.05 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:29:36.258 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Interval compaction: 0.00 GB write, 26.05 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:29:36.258 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-16T13:29:36.258 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.259 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: ** File Read Latency Histogram By Level [default] ** 2023-11-16T13:29:36.259 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.259 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: ** Compaction Stats [default] ** 2023-11-16T13:29:36.259 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:29:36.259 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:29:36.259 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: L0 2/0 77.41 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.7 0.00 0.00 1 0.001 0 0 2023-11-16T13:29:36.259 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Sum 2/0 77.41 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.7 0.00 0.00 1 0.001 0 0 2023-11-16T13:29:36.259 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-16T13:29:36.259 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.259 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: ** Compaction Stats [default] ** 2023-11-16T13:29:36.259 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:29:36.259 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:29:36.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 137.7 0.00 0.00 1 0.001 0 0 2023-11-16T13:29:36.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Uptime(secs): 0.0 total, 0.0 interval 2023-11-16T13:29:36.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Flush(GB): cumulative 0.000, interval 0.000 2023-11-16T13:29:36.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-16T13:29:36.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: AddFile(Total Files): cumulative 0, interval 0 2023-11-16T13:29:36.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: AddFile(L0 Files): cumulative 0, interval 0 2023-11-16T13:29:36.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: AddFile(Keys): cumulative 0, interval 0 2023-11-16T13:29:36.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Cumulative compaction: 0.00 GB write, 25.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:29:36.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:29:36.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-16T13:29:36.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: ** File Read Latency Histogram By Level [default] ** 2023-11-16T13:29:36.261 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.261 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.830+0000 7f8368b4e700 0 starting mon.smithi151 rank 0 at public addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] at bind addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi151 fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:29:36.261 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.831+0000 7f8368b4e700 1 mon.smithi151@-1(???) e1 preinit fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:29:36.261 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.831+0000 7f8368b4e700 0 mon.smithi151@-1(???).mds e1 new map 2023-11-16T13:29:36.261 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.831+0000 7f8368b4e700 0 mon.smithi151@-1(???).mds e1 print_map 2023-11-16T13:29:36.261 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: e1 2023-11-16T13:29:36.261 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: enable_multiple, ever_enabled_multiple: 1,1 2023-11-16T13:29:36.261 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-11-16T13:29:36.261 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: legacy client fscid: -1 2023-11-16T13:29:36.261 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.261 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: No filesystems configured 2023-11-16T13:29:36.261 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: 2023-11-16T13:29:36.262 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.831+0000 7f8368b4e700 0 mon.smithi151@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-11-16T13:29:36.262 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.831+0000 7f8368b4e700 0 mon.smithi151@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-11-16T13:29:36.262 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.831+0000 7f8368b4e700 0 mon.smithi151@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-11-16T13:29:36.262 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.831+0000 7f8368b4e700 0 mon.smithi151@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-11-16T13:29:36.262 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:34.123892+0000 mon.smithi151 (mon.0) 8 : audit [INF] from='client.? 172.21.15.151:0/2833170223' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-11-16T13:29:36.262 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:34.127311+0000 mon.smithi151 (mon.0) 9 : audit [INF] from='client.? 172.21.15.151:0/2833170223' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-11-16T13:29:36.262 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.831+0000 7f8368b4e700 1 mon.smithi151@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-11-16T13:29:36.262 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 systemd[1]: Started Ceph mon.smithi151 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:29:36.262 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.832+0000 7f8368b4e700 20 expand_channel_meta expand map: {default=false} 2023-11-16T13:29:36.262 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.832+0000 7f8368b4e700 20 expand_channel_meta from 'false' to 'false' 2023-11-16T13:29:36.262 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.832+0000 7f8368b4e700 20 expand_channel_meta expanded map: {default=false} 2023-11-16T13:29:36.263 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.832+0000 7f8368b4e700 20 expand_channel_meta expand map: {default=info} 2023-11-16T13:29:36.263 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.832+0000 7f8368b4e700 20 expand_channel_meta from 'info' to 'info' 2023-11-16T13:29:36.263 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.832+0000 7f8368b4e700 20 expand_channel_meta expanded map: {default=info} 2023-11-16T13:29:36.263 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.832+0000 7f8368b4e700 20 expand_channel_meta expand map: {default=daemon} 2023-11-16T13:29:36.263 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.832+0000 7f8368b4e700 20 expand_channel_meta from 'daemon' to 'daemon' 2023-11-16T13:29:36.263 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.832+0000 7f8368b4e700 20 expand_channel_meta expanded map: {default=daemon} 2023-11-16T13:29:36.263 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.832+0000 7f8368b4e700 20 expand_channel_meta expand map: {default=debug} 2023-11-16T13:29:36.263 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.832+0000 7f8368b4e700 20 expand_channel_meta from 'debug' to 'debug' 2023-11-16T13:29:36.263 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:29:35.832+0000 7f8368b4e700 20 expand_channel_meta expanded map: {default=debug} 2023-11-16T13:29:36.263 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:35.849693+0000 mon.smithi151 (mon.0) 1 : cluster [INF] mon.smithi151 is new leader, mons smithi151 in quorum (ranks 0) 2023-11-16T13:29:36.263 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:35.849791+0000 mon.smithi151 (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {smithi151=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0]} 2023-11-16T13:29:36.263 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:35.849858+0000 mon.smithi151 (mon.0) 3 : cluster [DBG] fsmap 2023-11-16T13:29:36.264 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:35.849900+0000 mon.smithi151 (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-11-16T13:29:36.264 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:35.850494+0000 mon.smithi151 (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-11-16T13:29:36.584 INFO:teuthology.orchestra.run.smithi151.stderr:Wrote config to /etc/ceph/ceph.conf 2023-11-16T13:29:36.584 INFO:teuthology.orchestra.run.smithi151.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-11-16T13:29:36.584 INFO:teuthology.orchestra.run.smithi151.stderr:Creating mgr... 2023-11-16T13:29:36.585 INFO:teuthology.orchestra.run.smithi151.stderr:Verifying port 9283 ... 2023-11-16T13:29:36.586 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl daemon-reload 2023-11-16T13:29:36.679 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl stop ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mgr.smithi151.fcujra 2023-11-16T13:29:36.707 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl reset-failed ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mgr.smithi151.fcujra 2023-11-16T13:29:36.712 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: Failed to reset failed state of unit ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mgr.smithi151.fcujra.service: Unit ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mgr.smithi151.fcujra.service not loaded. 2023-11-16T13:29:36.712 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl enable ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mgr.smithi151.fcujra 2023-11-16T13:29:36.719 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: Created symlink /etc/systemd/system/ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7.target.wants/ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mgr.smithi151.fcujra.service → /etc/systemd/system/ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@.service. 2023-11-16T13:29:36.803 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl start ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mgr.smithi151.fcujra 2023-11-16T13:29:37.129 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-enabled firewalld.service 2023-11-16T13:29:37.137 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: disabled 2023-11-16T13:29:37.137 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-active firewalld.service 2023-11-16T13:29:37.145 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: inactive 2023-11-16T13:29:37.145 INFO:teuthology.orchestra.run.smithi151.stderr:firewalld.service is not enabled 2023-11-16T13:29:37.146 INFO:teuthology.orchestra.run.smithi151.stderr:Not possible to enable service . firewalld.service is not available 2023-11-16T13:29:37.146 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-enabled firewalld.service 2023-11-16T13:29:37.153 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: disabled 2023-11-16T13:29:37.153 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-active firewalld.service 2023-11-16T13:29:37.161 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: inactive 2023-11-16T13:29:37.162 INFO:teuthology.orchestra.run.smithi151.stderr:firewalld.service is not enabled 2023-11-16T13:29:37.162 INFO:teuthology.orchestra.run.smithi151.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-11-16T13:29:37.162 INFO:teuthology.orchestra.run.smithi151.stderr:Waiting for mgr to start... 2023-11-16T13:29:37.162 INFO:teuthology.orchestra.run.smithi151.stderr:Waiting for mgr... 2023-11-16T13:29:37.162 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 status -f json-pretty 2023-11-16T13:29:37.392 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:36.229305+0000 mon.smithi151 (mon.0) 6 : audit [INF] from='client.? 172.21.15.151:0/374308385' entity='client.admin' 2023-11-16T13:29:37.688 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 2023-11-16T13:29:37.689 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: { 2023-11-16T13:29:37.689 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "fsid": "11fdb452-8484-11ee-8dbe-212e2dc638e7", 2023-11-16T13:29:37.689 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "health": { 2023-11-16T13:29:37.689 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-11-16T13:29:37.689 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "checks": {}, 2023-11-16T13:29:37.689 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "mutes": [] 2023-11-16T13:29:37.689 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:37.689 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-11-16T13:29:37.689 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "quorum": [ 2023-11-16T13:29:37.689 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 0 2023-11-16T13:29:37.689 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: ], 2023-11-16T13:29:37.689 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "quorum_names": [ 2023-11-16T13:29:37.690 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "smithi151" 2023-11-16T13:29:37.690 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: ], 2023-11-16T13:29:37.690 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "quorum_age": 1, 2023-11-16T13:29:37.690 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "monmap": { 2023-11-16T13:29:37.690 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-16T13:29:37.690 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-11-16T13:29:37.690 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_mons": 1 2023-11-16T13:29:37.690 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:37.690 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "osdmap": { 2023-11-16T13:29:37.690 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-16T13:29:37.690 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_osds": 0, 2023-11-16T13:29:37.691 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-11-16T13:29:37.691 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-11-16T13:29:37.694 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-11-16T13:29:37.694 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-11-16T13:29:37.694 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-11-16T13:29:37.694 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:37.694 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "pgmap": { 2023-11-16T13:29:37.694 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-11-16T13:29:37.694 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-11-16T13:29:37.694 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_pools": 0, 2023-11-16T13:29:37.694 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_objects": 0, 2023-11-16T13:29:37.695 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-11-16T13:29:37.695 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-11-16T13:29:37.695 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-11-16T13:29:37.695 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "bytes_total": 0 2023-11-16T13:29:37.695 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:37.695 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "fsmap": { 2023-11-16T13:29:37.695 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-16T13:29:37.695 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "by_rank": [], 2023-11-16T13:29:37.695 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "up:standby": 0 2023-11-16T13:29:37.695 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:37.695 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "mgrmap": { 2023-11-16T13:29:37.695 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "available": false, 2023-11-16T13:29:37.696 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-11-16T13:29:37.696 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "modules": [ 2023-11-16T13:29:37.696 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "iostat", 2023-11-16T13:29:37.696 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "nfs", 2023-11-16T13:29:37.696 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "restful" 2023-11-16T13:29:37.696 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: ], 2023-11-16T13:29:37.696 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "services": {} 2023-11-16T13:29:37.696 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:37.696 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "servicemap": { 2023-11-16T13:29:37.696 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-16T13:29:37.696 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "modified": "2023-11-16T13:29:32.865032+0000", 2023-11-16T13:29:37.696 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "services": {} 2023-11-16T13:29:37.697 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:37.697 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "progress_events": {} 2023-11-16T13:29:37.697 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: } 2023-11-16T13:29:38.034 INFO:teuthology.orchestra.run.smithi151.stderr:mgr not available, waiting (1/15)... 2023-11-16T13:29:38.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:37.685089+0000 mon.smithi151 (mon.0) 7 : audit [DBG] from='client.? 172.21.15.151:0/348286486' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-11-16T13:29:40.037 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 status -f json-pretty 2023-11-16T13:29:40.535 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 2023-11-16T13:29:40.535 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: { 2023-11-16T13:29:40.535 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "fsid": "11fdb452-8484-11ee-8dbe-212e2dc638e7", 2023-11-16T13:29:40.535 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "health": { 2023-11-16T13:29:40.535 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-11-16T13:29:40.535 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "checks": {}, 2023-11-16T13:29:40.535 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "mutes": [] 2023-11-16T13:29:40.536 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:40.536 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-11-16T13:29:40.536 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "quorum": [ 2023-11-16T13:29:40.536 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 0 2023-11-16T13:29:40.536 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: ], 2023-11-16T13:29:40.536 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "quorum_names": [ 2023-11-16T13:29:40.536 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "smithi151" 2023-11-16T13:29:40.536 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: ], 2023-11-16T13:29:40.536 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "quorum_age": 4, 2023-11-16T13:29:40.536 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "monmap": { 2023-11-16T13:29:40.536 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-16T13:29:40.536 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-11-16T13:29:40.536 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_mons": 1 2023-11-16T13:29:40.537 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:40.537 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "osdmap": { 2023-11-16T13:29:40.537 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-16T13:29:40.537 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_osds": 0, 2023-11-16T13:29:40.537 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-11-16T13:29:40.537 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-11-16T13:29:40.542 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-11-16T13:29:40.543 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-11-16T13:29:40.543 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-11-16T13:29:40.543 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:40.543 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "pgmap": { 2023-11-16T13:29:40.543 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-11-16T13:29:40.543 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-11-16T13:29:40.544 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_pools": 0, 2023-11-16T13:29:40.544 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_objects": 0, 2023-11-16T13:29:40.544 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-11-16T13:29:40.544 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-11-16T13:29:40.544 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-11-16T13:29:40.544 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "bytes_total": 0 2023-11-16T13:29:40.544 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:40.545 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "fsmap": { 2023-11-16T13:29:40.545 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-16T13:29:40.545 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "by_rank": [], 2023-11-16T13:29:40.545 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "up:standby": 0 2023-11-16T13:29:40.545 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:40.545 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "mgrmap": { 2023-11-16T13:29:40.545 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "available": false, 2023-11-16T13:29:40.545 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-11-16T13:29:40.546 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "modules": [ 2023-11-16T13:29:40.546 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "iostat", 2023-11-16T13:29:40.546 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "nfs", 2023-11-16T13:29:40.546 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "restful" 2023-11-16T13:29:40.546 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: ], 2023-11-16T13:29:40.547 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "services": {} 2023-11-16T13:29:40.547 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:40.547 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "servicemap": { 2023-11-16T13:29:40.547 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-16T13:29:40.547 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "modified": "2023-11-16T13:29:32.865032+0000", 2023-11-16T13:29:40.547 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "services": {} 2023-11-16T13:29:40.548 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:40.548 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "progress_events": {} 2023-11-16T13:29:40.548 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: } 2023-11-16T13:29:40.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:40.531303+0000 mon.smithi151 (mon.0) 8 : audit [DBG] from='client.? 172.21.15.151:0/4003473556' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-11-16T13:29:42.241 INFO:teuthology.orchestra.run.smithi151.stderr:mgr not available, waiting (2/15)... 2023-11-16T13:29:42.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:42.229640+0000 mon.smithi151 (mon.0) 9 : cluster [INF] Activating manager daemon smithi151.fcujra 2023-11-16T13:29:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:42.232235+0000 mon.smithi151 (mon.0) 10 : cluster [DBG] mgrmap e2: smithi151.fcujra(active, starting, since 0.00268167s) 2023-11-16T13:29:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:42.233120+0000 mon.smithi151 (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.151:0/2380085561' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-16T13:29:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:42.233301+0000 mon.smithi151 (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.151:0/2380085561' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-16T13:29:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:42.233460+0000 mon.smithi151 (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.151:0/2380085561' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-16T13:29:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:42.233622+0000 mon.smithi151 (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.151:0/2380085561' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:29:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:42.233783+0000 mon.smithi151 (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.151:0/2380085561' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi151.fcujra", "id": "smithi151.fcujra"}]: dispatch 2023-11-16T13:29:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:42.248350+0000 mon.smithi151 (mon.0) 16 : cluster [INF] Manager daemon smithi151.fcujra is now available 2023-11-16T13:29:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:42.258417+0000 mon.smithi151 (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.151:0/2380085561' entity='mgr.smithi151.fcujra' 2023-11-16T13:29:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:42.261654+0000 mon.smithi151 (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.151:0/2380085561' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:29:42.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:42.262804+0000 mon.smithi151 (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.151:0/2380085561' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:29:42.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:42.267885+0000 mon.smithi151 (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.151:0/2380085561' entity='mgr.smithi151.fcujra' 2023-11-16T13:29:42.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:42.273345+0000 mon.smithi151 (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.151:0/2380085561' entity='mgr.smithi151.fcujra' 2023-11-16T13:29:44.241 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 status -f json-pretty 2023-11-16T13:29:44.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:43.235268+0000 mon.smithi151 (mon.0) 22 : cluster [DBG] mgrmap e3: smithi151.fcujra(active, since 1.00571s) 2023-11-16T13:29:44.803 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 2023-11-16T13:29:44.804 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: { 2023-11-16T13:29:44.804 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "fsid": "11fdb452-8484-11ee-8dbe-212e2dc638e7", 2023-11-16T13:29:44.804 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "health": { 2023-11-16T13:29:44.804 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-11-16T13:29:44.804 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "checks": {}, 2023-11-16T13:29:44.804 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "mutes": [] 2023-11-16T13:29:44.804 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:44.804 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-11-16T13:29:44.804 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "quorum": [ 2023-11-16T13:29:44.805 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 0 2023-11-16T13:29:44.805 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: ], 2023-11-16T13:29:44.805 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "quorum_names": [ 2023-11-16T13:29:44.805 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "smithi151" 2023-11-16T13:29:44.805 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: ], 2023-11-16T13:29:44.805 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "quorum_age": 8, 2023-11-16T13:29:44.805 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "monmap": { 2023-11-16T13:29:44.805 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-16T13:29:44.805 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-11-16T13:29:44.805 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_mons": 1 2023-11-16T13:29:44.805 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:44.806 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "osdmap": { 2023-11-16T13:29:44.806 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-16T13:29:44.806 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_osds": 0, 2023-11-16T13:29:44.806 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-11-16T13:29:44.806 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-11-16T13:29:44.809 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-11-16T13:29:44.809 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-11-16T13:29:44.809 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-11-16T13:29:44.809 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:44.810 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "pgmap": { 2023-11-16T13:29:44.810 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-11-16T13:29:44.810 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-11-16T13:29:44.810 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_pools": 0, 2023-11-16T13:29:44.810 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_objects": 0, 2023-11-16T13:29:44.810 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-11-16T13:29:44.810 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-11-16T13:29:44.810 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-11-16T13:29:44.810 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "bytes_total": 0 2023-11-16T13:29:44.810 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:44.810 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "fsmap": { 2023-11-16T13:29:44.810 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-16T13:29:44.811 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "by_rank": [], 2023-11-16T13:29:44.811 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "up:standby": 0 2023-11-16T13:29:44.811 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:44.811 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "mgrmap": { 2023-11-16T13:29:44.811 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "available": true, 2023-11-16T13:29:44.811 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-11-16T13:29:44.811 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "modules": [ 2023-11-16T13:29:44.811 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "iostat", 2023-11-16T13:29:44.811 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "nfs", 2023-11-16T13:29:44.812 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "restful" 2023-11-16T13:29:44.812 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: ], 2023-11-16T13:29:44.812 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "services": {} 2023-11-16T13:29:44.812 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:44.812 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "servicemap": { 2023-11-16T13:29:44.812 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-16T13:29:44.812 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "modified": "2023-11-16T13:29:32.865032+0000", 2023-11-16T13:29:44.812 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "services": {} 2023-11-16T13:29:44.812 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: }, 2023-11-16T13:29:44.812 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "progress_events": {} 2023-11-16T13:29:44.813 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: } 2023-11-16T13:29:45.158 INFO:teuthology.orchestra.run.smithi151.stderr:mgr is available 2023-11-16T13:29:45.158 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z -v /tmp/tmpf9r8tib1:/var/lib/ceph/user.conf:z docker.io/ceph/ceph:v16.2.5 config assimilate-conf -i /var/lib/ceph/user.conf 2023-11-16T13:29:45.339 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:44.800439+0000 mon.smithi151 (mon.0) 23 : audit [DBG] from='client.? 172.21.15.151:0/4074526438' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-11-16T13:29:45.676 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 2023-11-16T13:29:45.676 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: [global] 2023-11-16T13:29:45.676 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: fsid = 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:29:45.676 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: log_to_journald = false 2023-11-16T13:29:45.676 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-11-16T13:29:45.676 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-11-16T13:29:45.676 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-11-16T13:29:45.676 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-11-16T13:29:45.676 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 2023-11-16T13:29:45.676 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: [mgr] 2023-11-16T13:29:45.676 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-11-16T13:29:45.677 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 2023-11-16T13:29:45.677 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: [osd] 2023-11-16T13:29:45.677 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-11-16T13:29:45.677 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-11-16T13:29:46.016 INFO:teuthology.orchestra.run.smithi151.stderr:Enabling cephadm module... 2023-11-16T13:29:46.016 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 mgr module enable cephadm 2023-11-16T13:29:46.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:45.235155+0000 mon.smithi151 (mon.0) 24 : cluster [DBG] mgrmap e4: smithi151.fcujra(active, since 3s) 2023-11-16T13:29:46.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:45.672993+0000 mon.smithi151 (mon.0) 25 : audit [INF] from='client.? 172.21.15.151:0/2478973900' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-11-16T13:29:47.526 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:46.548498+0000 mon.smithi151 (mon.0) 26 : audit [INF] from='client.? 172.21.15.151:0/1781083852' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-11-16T13:29:47.590 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 mgr stat 2023-11-16T13:29:48.608 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: { 2023-11-16T13:29:48.609 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 5, 2023-11-16T13:29:48.609 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "available": true, 2023-11-16T13:29:48.609 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "active_name": "smithi151.fcujra", 2023-11-16T13:29:48.609 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_standby": 0 2023-11-16T13:29:48.609 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: } 2023-11-16T13:29:48.631 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:47.241364+0000 mon.smithi151 (mon.0) 27 : audit [INF] from='client.? 172.21.15.151:0/1781083852' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-11-16T13:29:48.632 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:47.241420+0000 mon.smithi151 (mon.0) 28 : cluster [DBG] mgrmap e5: smithi151.fcujra(active, since 5s) 2023-11-16T13:29:49.583 INFO:teuthology.orchestra.run.smithi151.stderr:Waiting for the mgr to restart... 2023-11-16T13:29:49.584 INFO:teuthology.orchestra.run.smithi151.stderr:Waiting for mgr epoch 5... 2023-11-16T13:29:49.584 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 tell mgr mgr_status 2023-11-16T13:29:49.587 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:48.606919+0000 mon.smithi151 (mon.0) 29 : audit [DBG] from='client.? 172.21.15.151:0/1279658130' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-11-16T13:29:52.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:52.404681+0000 mon.smithi151 (mon.0) 30 : cluster [INF] Active manager daemon smithi151.fcujra restarted 2023-11-16T13:29:52.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:52.405053+0000 mon.smithi151 (mon.0) 31 : cluster [INF] Activating manager daemon smithi151.fcujra 2023-11-16T13:29:52.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:52.408990+0000 mon.smithi151 (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-11-16T13:29:53.479 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: { 2023-11-16T13:29:53.479 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-11-16T13:29:53.479 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "initialized": true 2023-11-16T13:29:53.480 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: } 2023-11-16T13:29:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:52.460062+0000 mon.smithi151 (mon.0) 33 : cluster [DBG] mgrmap e6: smithi151.fcujra(active, starting, since 0.0550814s) 2023-11-16T13:29:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.462469+0000 mon.smithi151 (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:29:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.463095+0000 mon.smithi151 (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi151.fcujra", "id": "smithi151.fcujra"}]: dispatch 2023-11-16T13:29:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.464119+0000 mon.smithi151 (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-16T13:29:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.464336+0000 mon.smithi151 (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-16T13:29:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.464536+0000 mon.smithi151 (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-16T13:29:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:52.478575+0000 mon.smithi151 (mon.0) 39 : cluster [INF] Manager daemon smithi151.fcujra is now available 2023-11-16T13:29:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.491906+0000 mon.smithi151 (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' 2023-11-16T13:29:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.494578+0000 mon.smithi151 (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' 2023-11-16T13:29:53.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.497110+0000 mon.smithi151 (mon.0) 42 : audit [INF] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' 2023-11-16T13:29:53.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.586249+0000 mon.smithi151 (mon.0) 43 : audit [INF] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' 2023-11-16T13:29:53.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.586908+0000 mon.smithi151 (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:29:53.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.587883+0000 mon.smithi151 (mon.0) 45 : audit [DBG] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:29:53.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.589681+0000 mon.smithi151 (mon.0) 46 : audit [DBG] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:29:53.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.600388+0000 mon.smithi151 (mon.0) 47 : audit [INF] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:29:53.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:52.601500+0000 mon.smithi151 (mon.0) 48 : audit [INF] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:29:54.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:54 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:53.469447+0000 mgr.smithi151.fcujra (mgr.14116) 1 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-11-16T13:29:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:54 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:53.470227+0000 mon.smithi151 (mon.0) 49 : cluster [DBG] mgrmap e7: smithi151.fcujra(active, since 1.06524s) 2023-11-16T13:29:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:54 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:53.476513+0000 mgr.smithi151.fcujra (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-11-16T13:29:55.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:55 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:29:54.472450+0000 mon.smithi151 (mon.0) 50 : cluster [DBG] mgrmap e8: smithi151.fcujra(active, since 2s) 2023-11-16T13:29:56.123 INFO:teuthology.orchestra.run.smithi151.stderr:mgr epoch 5 is available 2023-11-16T13:29:56.123 INFO:teuthology.orchestra.run.smithi151.stderr:Setting orchestrator backend to cephadm... 2023-11-16T13:29:56.124 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch set backend cephadm 2023-11-16T13:29:57.014 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 cephadm set-user root 2023-11-16T13:29:57.495 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: value unchanged 2023-11-16T13:29:57.774 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:56.645946+0000 mgr.smithi151.fcujra (mgr.14116) 3 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:29:57.774 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:56.649773+0000 mon.smithi151 (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' 2023-11-16T13:29:57.775 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:56.653390+0000 mon.smithi151 (mon.0) 52 : audit [DBG] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:29:57.839 INFO:teuthology.orchestra.run.smithi151.stderr:Generating ssh key... 2023-11-16T13:29:57.839 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 cephadm generate-key 2023-11-16T13:29:58.941 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:57.493770+0000 mgr.smithi151.fcujra (mgr.14116) 4 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:30:05.682 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:58.648334+0000 mon.smithi151 (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:05.683 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 cephadm get-pub-key 2023-11-16T13:30:05.683 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDb1Eu/4AGq+ohNgQr47vJN5TSQLzMUKddvbFi0IIONsxZKtW7xLGa0OgCK3zyI70d6pkpnEVVtqvL8GxfLZaGNw7fuuM71c6EYVc3VDKqmyMp3rKrP49fHDt0PM1UQLhrpqOP7ERFJcm2olEyhydm1NU6AaB1yOWPBhgd4QrSSwitwXAD+/yKDvyQVHJ1ZCfCbIjIKY3QFe18+XAXEBg6deWBj2+6cjg4xtFcz1gg+VlLmUSQgVWDsXSx1Ii11tKz7UMPpWC8DvkYEBOgT06ifOOraq8L+Jq+nPDjUXYujuuApancot2EvcO+UdSNe59FgljsvDfGspChZOeYsFagIv1tOzhaQHr5DIP1vFn7ViVnm2PpXoOQtGffB5/3iGNUJLycwg8ImFyFc9POroZtvI0ncGFBl5WwggF8OaqWPmw07Bhl11+CdJb/LeTSk8xmPKfs3CFP1eOQW28ADn1FoHBoFCFFGjcJaEFxFghi7naMNT4R3wUWGUOwcP/wABMs= ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:30:05.683 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:29:58.380639+0000 mgr.smithi151.fcujra (mgr.14116) 6 : cephadm [INF] Generating ssh key... 2023-11-16T13:30:05.683 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:29:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:58.652591+0000 mon.smithi151 (mon.0) 54 : audit [INF] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:05.684 INFO:teuthology.orchestra.run.smithi151.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-11-16T13:30:05.741 INFO:teuthology.orchestra.run.smithi151.stderr:Adding key to root@localhost authorized_keys... 2023-11-16T13:30:05.741 INFO:teuthology.orchestra.run.smithi151.stderr:Adding host smithi151... 2023-11-16T13:30:05.742 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch host add smithi151 172.21.15.151 2023-11-16T13:30:05.742 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: Added host 'smithi151' with addr '172.21.15.151' 2023-11-16T13:30:05.742 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:29:59.492474+0000 mgr.smithi151.fcujra (mgr.14116) 7 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:30:05.743 INFO:teuthology.orchestra.run.smithi151.stderr:Deploying mon service with default placement... 2023-11-16T13:30:05.743 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply mon 2023-11-16T13:30:05.760 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: Scheduled mon update... 2023-11-16T13:30:05.760 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:00.365794+0000 mgr.smithi151.fcujra (mgr.14116) 8 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi151", "addr": "172.21.15.151", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:30:05.760 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:00.955961+0000 mon.smithi151 (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:05.760 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:00.956231+0000 mgr.smithi151.fcujra (mgr.14116) 9 : cephadm [INF] Added host smithi151 2023-11-16T13:30:05.761 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:00.956483+0000 mon.smithi151 (mon.0) 56 : audit [DBG] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:30:05.761 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:01.769109+0000 mon.smithi151 (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:05.761 INFO:teuthology.orchestra.run.smithi151.stderr:Deploying mgr service with default placement... 2023-11-16T13:30:05.762 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply mgr 2023-11-16T13:30:05.762 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-11-16T13:30:05.762 INFO:teuthology.orchestra.run.smithi151.stderr:Deploying crash service with default placement... 2023-11-16T13:30:05.762 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply crash 2023-11-16T13:30:05.762 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:01.766212+0000 mgr.smithi151.fcujra (mgr.14116) 10 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:30:05.762 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:01.766940+0000 mgr.smithi151.fcujra (mgr.14116) 11 : cephadm [INF] Saving service mon spec with placement count:5 2023-11-16T13:30:05.762 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:02.674170+0000 mon.smithi151 (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:05.763 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: Scheduled crash update... 2023-11-16T13:30:05.763 INFO:teuthology.orchestra.run.smithi151.stderr:Enabling mgr prometheus module... 2023-11-16T13:30:05.763 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 mgr module enable prometheus 2023-11-16T13:30:05.765 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:02.671401+0000 mgr.smithi151.fcujra (mgr.14116) 12 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:30:05.765 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:02.671937+0000 mgr.smithi151.fcujra (mgr.14116) 13 : cephadm [INF] Saving service mgr spec with placement count:2 2023-11-16T13:30:05.765 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:03.541889+0000 mon.smithi151 (mon.0) 59 : audit [INF] from='mgr.14116 172.21.15.151:0/3176194158' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:05.765 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:03.539115+0000 mgr.smithi151.fcujra (mgr.14116) 14 : audit [DBG] from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:30:05.765 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:03.539767+0000 mgr.smithi151.fcujra (mgr.14116) 15 : cephadm [INF] Saving service crash spec with placement * 2023-11-16T13:30:05.765 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:04.420574+0000 mon.smithi151 (mon.0) 60 : audit [INF] from='client.? 172.21.15.151:0/2560458163' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-11-16T13:30:05.765 INFO:teuthology.orchestra.run.smithi151.stderr:Deploying prometheus service with default placement... 2023-11-16T13:30:05.765 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply prometheus 2023-11-16T13:30:06.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:04.965126+0000 mon.smithi151 (mon.0) 61 : audit [INF] from='client.? 172.21.15.151:0/2560458163' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-11-16T13:30:06.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:04.965173+0000 mon.smithi151 (mon.0) 62 : cluster [DBG] mgrmap e9: smithi151.fcujra(active, since 12s) 2023-11-16T13:30:10.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:10.106460+0000 mon.smithi151 (mon.0) 63 : cluster [INF] Active manager daemon smithi151.fcujra restarted 2023-11-16T13:30:10.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:10.106872+0000 mon.smithi151 (mon.0) 64 : cluster [INF] Activating manager daemon smithi151.fcujra 2023-11-16T13:30:10.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:10.109182+0000 mon.smithi151 (mon.0) 65 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-11-16T13:30:11.169 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: Scheduled prometheus update... 2023-11-16T13:30:11.448 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:10.159845+0000 mon.smithi151 (mon.0) 66 : cluster [DBG] mgrmap e10: smithi151.fcujra(active, starting, since 0.0530679s) 2023-11-16T13:30:11.448 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:10.162276+0000 mon.smithi151 (mon.0) 67 : audit [DBG] from='mgr.14144 172.21.15.151:0/2933794246' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:30:11.448 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:10.162824+0000 mon.smithi151 (mon.0) 68 : audit [DBG] from='mgr.14144 172.21.15.151:0/2933794246' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi151.fcujra", "id": "smithi151.fcujra"}]: dispatch 2023-11-16T13:30:11.448 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:10.163535+0000 mon.smithi151 (mon.0) 69 : audit [DBG] from='mgr.14144 172.21.15.151:0/2933794246' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-16T13:30:11.448 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:10.163684+0000 mon.smithi151 (mon.0) 70 : audit [DBG] from='mgr.14144 172.21.15.151:0/2933794246' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-16T13:30:11.448 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:10.163823+0000 mon.smithi151 (mon.0) 71 : audit [DBG] from='mgr.14144 172.21.15.151:0/2933794246' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-16T13:30:11.449 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:10.165633+0000 mon.smithi151 (mon.0) 72 : cluster [INF] Manager daemon smithi151.fcujra is now available 2023-11-16T13:30:11.449 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:10.262361+0000 mon.smithi151 (mon.0) 73 : audit [DBG] from='mgr.14144 172.21.15.151:0/2933794246' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:30:11.449 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:10.263424+0000 mon.smithi151 (mon.0) 74 : audit [DBG] from='mgr.14144 172.21.15.151:0/2933794246' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:30:11.449 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:10.279393+0000 mon.smithi151 (mon.0) 75 : audit [INF] from='mgr.14144 172.21.15.151:0/2933794246' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:30:11.449 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:10.287766+0000 mon.smithi151 (mon.0) 76 : audit [INF] from='mgr.14144 172.21.15.151:0/2933794246' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:30:11.513 INFO:teuthology.orchestra.run.smithi151.stderr:Deploying grafana service with default placement... 2023-11-16T13:30:11.513 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply grafana 2023-11-16T13:30:12.041 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: Scheduled grafana update... 2023-11-16T13:30:12.386 INFO:teuthology.orchestra.run.smithi151.stderr:Deploying node-exporter service with default placement... 2023-11-16T13:30:12.387 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply node-exporter 2023-11-16T13:30:12.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:11.163889+0000 mon.smithi151 (mon.0) 77 : cluster [DBG] mgrmap e11: smithi151.fcujra(active, since 1.05711s) 2023-11-16T13:30:12.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:11.164871+0000 mgr.smithi151.fcujra (mgr.14144) 2 : cephadm [INF] Saving service prometheus spec with placement count:1 2023-11-16T13:30:12.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:11.167451+0000 mon.smithi151 (mon.0) 78 : audit [INF] from='mgr.14144 172.21.15.151:0/2933794246' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:12.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:12.040153+0000 mon.smithi151 (mon.0) 79 : audit [INF] from='mgr.14144 172.21.15.151:0/2933794246' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:12.911 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2023-11-16T13:30:13.270 INFO:teuthology.orchestra.run.smithi151.stderr:Deploying alertmanager service with default placement... 2023-11-16T13:30:13.270 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply alertmanager 2023-11-16T13:30:13.778 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2023-11-16T13:30:14.111 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 config set mgr mgr/cephadm/container_init True --force 2023-11-16T13:30:14.207 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:12.037444+0000 mgr.smithi151.fcujra (mgr.14144) 3 : audit [DBG] from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:30:14.207 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:12.037957+0000 mgr.smithi151.fcujra (mgr.14144) 4 : cephadm [INF] Saving service grafana spec with placement count:1 2023-11-16T13:30:14.208 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:12.907142+0000 mgr.smithi151.fcujra (mgr.14144) 5 : audit [DBG] from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:30:14.208 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:12.907850+0000 mgr.smithi151.fcujra (mgr.14144) 6 : cephadm [INF] Saving service node-exporter spec with placement * 2023-11-16T13:30:14.208 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:12.909964+0000 mon.smithi151 (mon.0) 80 : audit [INF] from='mgr.14144 172.21.15.151:0/2933794246' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:14.208 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:13.041814+0000 mon.smithi151 (mon.0) 81 : cluster [DBG] mgrmap e12: smithi151.fcujra(active, since 2s) 2023-11-16T13:30:14.208 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:13.776881+0000 mon.smithi151 (mon.0) 82 : audit [INF] from='mgr.14144 172.21.15.151:0/2933794246' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:14.961 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 config set mgr mgr/dashboard/ssl_server_port 8443 2023-11-16T13:30:15.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:13.774159+0000 mgr.smithi151.fcujra (mgr.14144) 7 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:30:15.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:13.774681+0000 mgr.smithi151.fcujra (mgr.14144) 8 : cephadm [INF] Saving service alertmanager spec with placement count:1 2023-11-16T13:30:15.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:14.611627+0000 mon.smithi151 (mon.0) 83 : audit [INF] from='client.? 172.21.15.151:0/3525864893' entity='client.admin' 2023-11-16T13:30:15.811 INFO:teuthology.orchestra.run.smithi151.stderr:Enabling the dashboard module... 2023-11-16T13:30:15.812 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 mgr module enable dashboard 2023-11-16T13:30:16.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:15.463849+0000 mon.smithi151 (mon.0) 84 : audit [INF] from='client.? 172.21.15.151:0/705265873' entity='client.admin' 2023-11-16T13:30:16.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:16.362034+0000 mon.smithi151 (mon.0) 85 : audit [INF] from='client.? 172.21.15.151:0/3600963463' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-11-16T13:30:17.219 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 mgr stat 2023-11-16T13:30:17.930 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: { 2023-11-16T13:30:17.930 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "epoch": 13, 2023-11-16T13:30:17.930 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "available": true, 2023-11-16T13:30:17.930 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "active_name": "smithi151.fcujra", 2023-11-16T13:30:17.930 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "num_standby": 0 2023-11-16T13:30:17.930 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: } 2023-11-16T13:30:18.071 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:16.845178+0000 mon.smithi151 (mon.0) 86 : audit [INF] from='client.? 172.21.15.151:0/3600963463' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-11-16T13:30:18.072 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:16.845224+0000 mon.smithi151 (mon.0) 87 : cluster [DBG] mgrmap e13: smithi151.fcujra(active, since 6s) 2023-11-16T13:30:18.403 INFO:teuthology.orchestra.run.smithi151.stderr:Waiting for the mgr to restart... 2023-11-16T13:30:18.404 INFO:teuthology.orchestra.run.smithi151.stderr:Waiting for mgr epoch 13... 2023-11-16T13:30:18.404 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 tell mgr mgr_status 2023-11-16T13:30:19.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:17.926371+0000 mon.smithi151 (mon.0) 88 : audit [DBG] from='client.? 172.21.15.151:0/605498401' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-11-16T13:30:22.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:22.036958+0000 mon.smithi151 (mon.0) 89 : cluster [INF] Active manager daemon smithi151.fcujra restarted 2023-11-16T13:30:22.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:22.037357+0000 mon.smithi151 (mon.0) 90 : cluster [INF] Activating manager daemon smithi151.fcujra 2023-11-16T13:30:22.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:22.041772+0000 mon.smithi151 (mon.0) 91 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-11-16T13:30:23.108 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: { 2023-11-16T13:30:23.108 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "mgrmap_epoch": 15, 2023-11-16T13:30:23.108 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: "initialized": true 2023-11-16T13:30:23.109 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: } 2023-11-16T13:30:23.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:22.092835+0000 mon.smithi151 (mon.0) 92 : cluster [DBG] mgrmap e14: smithi151.fcujra(active, starting, since 0.0555632s) 2023-11-16T13:30:23.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:22.096027+0000 mon.smithi151 (mon.0) 93 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:30:23.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:22.096889+0000 mon.smithi151 (mon.0) 94 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi151.fcujra", "id": "smithi151.fcujra"}]: dispatch 2023-11-16T13:30:23.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:22.097579+0000 mon.smithi151 (mon.0) 95 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-16T13:30:23.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:22.097738+0000 mon.smithi151 (mon.0) 96 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-16T13:30:23.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:22.097860+0000 mon.smithi151 (mon.0) 97 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-16T13:30:23.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:22.111271+0000 mon.smithi151 (mon.0) 98 : cluster [INF] Manager daemon smithi151.fcujra is now available 2023-11-16T13:30:23.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:22.210420+0000 mon.smithi151 (mon.0) 99 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:30:23.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:22.212495+0000 mon.smithi151 (mon.0) 100 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:30:23.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:22.237375+0000 mon.smithi151 (mon.0) 101 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:30:23.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:22.239806+0000 mon.smithi151 (mon.0) 102 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:30:23.613 INFO:teuthology.orchestra.run.smithi151.stderr:mgr epoch 13 is available 2023-11-16T13:30:23.613 INFO:teuthology.orchestra.run.smithi151.stderr:Generating a dashboard self-signed certificate... 2023-11-16T13:30:23.613 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 dashboard create-self-signed-cert 2023-11-16T13:30:24.267 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: Self-signed certificate created 2023-11-16T13:30:24.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:23.098285+0000 mgr.smithi151.fcujra (mgr.14164) 1 : audit [DBG] from='client.14168 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-11-16T13:30:24.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:23.099178+0000 mon.smithi151 (mon.0) 103 : cluster [DBG] mgrmap e15: smithi151.fcujra(active, since 1.06191s) 2023-11-16T13:30:24.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:23.105696+0000 mgr.smithi151.fcujra (mgr.14164) 2 : audit [DBG] from='client.14168 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-11-16T13:30:24.618 INFO:teuthology.orchestra.run.smithi151.stderr:Creating initial admin user... 2023-11-16T13:30:24.619 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z -v /tmp/ceph-tmpg5j6nef7:/tmp/dashboard.pw:z docker.io/ceph/ceph:v16.2.5 dashboard ac-user-create admin -i /tmp/dashboard.pw administrator --force-password --pwd-update-required 2023-11-16T13:30:25.359 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$HnTFiewGiHHtIAyYvNIO/ezlYj.uIc.prtrYqeO3PrDjtmIsHV7RG", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1700141425, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-11-16T13:30:25.389 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:25 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:24.104497+0000 mon.smithi151 (mon.0) 104 : cluster [DBG] mgrmap e16: smithi151.fcujra(active, since 2s) 2023-11-16T13:30:25.389 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:25 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:24.148963+0000 mgr.smithi151.fcujra (mgr.14164) 3 : audit [DBG] from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:30:25.389 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:25 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:24.260171+0000 mon.smithi151 (mon.0) 105 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:25.390 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:25 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:24.265113+0000 mon.smithi151 (mon.0) 106 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:25.710 INFO:teuthology.orchestra.run.smithi151.stderr:Fetching dashboard port number... 2023-11-16T13:30:25.710 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi151 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmphkobbpwh:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjkhnvsed:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 config get mgr mgr/dashboard/ssl_server_port 2023-11-16T13:30:26.217 INFO:teuthology.orchestra.run.smithi151.stderr:/usr/bin/ceph: 8443 2023-11-16T13:30:26.495 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:25.140270+0000 mgr.smithi151.fcujra (mgr.14164) 4 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:30:26.495 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:25.357911+0000 mon.smithi151 (mon.0) 107 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:26.495 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:26.215592+0000 mon.smithi151 (mon.0) 108 : audit [DBG] from='client.? 172.21.15.151:0/3070881071' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-11-16T13:30:26.567 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-enabled firewalld.service 2023-11-16T13:30:26.576 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: disabled 2023-11-16T13:30:26.576 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: systemctl is-active firewalld.service 2023-11-16T13:30:26.583 INFO:teuthology.orchestra.run.smithi151.stderr:systemctl: inactive 2023-11-16T13:30:26.583 INFO:teuthology.orchestra.run.smithi151.stderr:firewalld.service is not enabled 2023-11-16T13:30:26.583 INFO:teuthology.orchestra.run.smithi151.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-11-16T13:30:26.586 INFO:teuthology.orchestra.run.smithi151.stderr:Ceph Dashboard is now available at: 2023-11-16T13:30:26.586 INFO:teuthology.orchestra.run.smithi151.stderr: 2023-11-16T13:30:26.586 INFO:teuthology.orchestra.run.smithi151.stderr: URL: https://smithi151.front.sepia.ceph.com:8443/ 2023-11-16T13:30:26.587 INFO:teuthology.orchestra.run.smithi151.stderr: User: admin 2023-11-16T13:30:26.587 INFO:teuthology.orchestra.run.smithi151.stderr: Password: 8xryvwvu6z 2023-11-16T13:30:26.587 INFO:teuthology.orchestra.run.smithi151.stderr: 2023-11-16T13:30:26.587 INFO:teuthology.orchestra.run.smithi151.stderr:You can access the Ceph CLI with: 2023-11-16T13:30:26.587 INFO:teuthology.orchestra.run.smithi151.stderr: 2023-11-16T13:30:26.587 INFO:teuthology.orchestra.run.smithi151.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-11-16T13:30:26.587 INFO:teuthology.orchestra.run.smithi151.stderr: 2023-11-16T13:30:26.587 INFO:teuthology.orchestra.run.smithi151.stderr:Please consider enabling telemetry to help improve Ceph: 2023-11-16T13:30:26.587 INFO:teuthology.orchestra.run.smithi151.stderr: 2023-11-16T13:30:26.587 INFO:teuthology.orchestra.run.smithi151.stderr: ceph telemetry on 2023-11-16T13:30:26.587 INFO:teuthology.orchestra.run.smithi151.stderr: 2023-11-16T13:30:26.587 INFO:teuthology.orchestra.run.smithi151.stderr:For more information see: 2023-11-16T13:30:26.587 INFO:teuthology.orchestra.run.smithi151.stderr: 2023-11-16T13:30:26.588 INFO:teuthology.orchestra.run.smithi151.stderr: https://docs.ceph.com/docs/pacific/mgr/telemetry/ 2023-11-16T13:30:26.588 INFO:teuthology.orchestra.run.smithi151.stderr: 2023-11-16T13:30:26.588 INFO:teuthology.orchestra.run.smithi151.stderr:Bootstrap complete. 2023-11-16T13:30:26.588 INFO:teuthology.orchestra.run.smithi151.stderr:Releasing lock 139848003272832 on /run/cephadm/11fdb452-8484-11ee-8dbe-212e2dc638e7.lock 2023-11-16T13:30:26.588 INFO:teuthology.orchestra.run.smithi151.stderr:Lock 139848003272832 released on /run/cephadm/11fdb452-8484-11ee-8dbe-212e2dc638e7.lock 2023-11-16T13:30:26.618 INFO:tasks.cephadm:Fetching config... 2023-11-16T13:30:26.618 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-11-16T13:30:26.618 DEBUG:teuthology.orchestra.run.smithi151:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-11-16T13:30:26.636 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-11-16T13:30:26.636 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-11-16T13:30:26.636 DEBUG:teuthology.orchestra.run.smithi151:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-11-16T13:30:26.694 INFO:tasks.cephadm:Fetching mon keyring... 2023-11-16T13:30:26.694 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-11-16T13:30:26.695 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/keyring of=/dev/stdout 2023-11-16T13:30:26.762 INFO:tasks.cephadm:Fetching pub ssh key... 2023-11-16T13:30:26.762 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-11-16T13:30:26.762 DEBUG:teuthology.orchestra.run.smithi151:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-11-16T13:30:26.820 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-11-16T13:30:26.820 DEBUG:teuthology.orchestra.run.smithi151:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDb1Eu/4AGq+ohNgQr47vJN5TSQLzMUKddvbFi0IIONsxZKtW7xLGa0OgCK3zyI70d6pkpnEVVtqvL8GxfLZaGNw7fuuM71c6EYVc3VDKqmyMp3rKrP49fHDt0PM1UQLhrpqOP7ERFJcm2olEyhydm1NU6AaB1yOWPBhgd4QrSSwitwXAD+/yKDvyQVHJ1ZCfCbIjIKY3QFe18+XAXEBg6deWBj2+6cjg4xtFcz1gg+VlLmUSQgVWDsXSx1Ii11tKz7UMPpWC8DvkYEBOgT06ifOOraq8L+Jq+nPDjUXYujuuApancot2EvcO+UdSNe59FgljsvDfGspChZOeYsFagIv1tOzhaQHr5DIP1vFn7ViVnm2PpXoOQtGffB5/3iGNUJLycwg8ImFyFc9POroZtvI0ncGFBl5WwggF8OaqWPmw07Bhl11+CdJb/LeTSk8xmPKfs3CFP1eOQW28ADn1FoHBoFCFFGjcJaEFxFghi7naMNT4R3wUWGUOwcP/wABMs= ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-11-16T13:30:26.895 INFO:teuthology.orchestra.run.smithi151.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDb1Eu/4AGq+ohNgQr47vJN5TSQLzMUKddvbFi0IIONsxZKtW7xLGa0OgCK3zyI70d6pkpnEVVtqvL8GxfLZaGNw7fuuM71c6EYVc3VDKqmyMp3rKrP49fHDt0PM1UQLhrpqOP7ERFJcm2olEyhydm1NU6AaB1yOWPBhgd4QrSSwitwXAD+/yKDvyQVHJ1ZCfCbIjIKY3QFe18+XAXEBg6deWBj2+6cjg4xtFcz1gg+VlLmUSQgVWDsXSx1Ii11tKz7UMPpWC8DvkYEBOgT06ifOOraq8L+Jq+nPDjUXYujuuApancot2EvcO+UdSNe59FgljsvDfGspChZOeYsFagIv1tOzhaQHr5DIP1vFn7ViVnm2PpXoOQtGffB5/3iGNUJLycwg8ImFyFc9POroZtvI0ncGFBl5WwggF8OaqWPmw07Bhl11+CdJb/LeTSk8xmPKfs3CFP1eOQW28ADn1FoHBoFCFFGjcJaEFxFghi7naMNT4R3wUWGUOwcP/wABMs= ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:30:26.906 DEBUG:teuthology.orchestra.run.smithi167:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDb1Eu/4AGq+ohNgQr47vJN5TSQLzMUKddvbFi0IIONsxZKtW7xLGa0OgCK3zyI70d6pkpnEVVtqvL8GxfLZaGNw7fuuM71c6EYVc3VDKqmyMp3rKrP49fHDt0PM1UQLhrpqOP7ERFJcm2olEyhydm1NU6AaB1yOWPBhgd4QrSSwitwXAD+/yKDvyQVHJ1ZCfCbIjIKY3QFe18+XAXEBg6deWBj2+6cjg4xtFcz1gg+VlLmUSQgVWDsXSx1Ii11tKz7UMPpWC8DvkYEBOgT06ifOOraq8L+Jq+nPDjUXYujuuApancot2EvcO+UdSNe59FgljsvDfGspChZOeYsFagIv1tOzhaQHr5DIP1vFn7ViVnm2PpXoOQtGffB5/3iGNUJLycwg8ImFyFc9POroZtvI0ncGFBl5WwggF8OaqWPmw07Bhl11+CdJb/LeTSk8xmPKfs3CFP1eOQW28ADn1FoHBoFCFFGjcJaEFxFghi7naMNT4R3wUWGUOwcP/wABMs= ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-11-16T13:30:26.955 INFO:teuthology.orchestra.run.smithi167.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDb1Eu/4AGq+ohNgQr47vJN5TSQLzMUKddvbFi0IIONsxZKtW7xLGa0OgCK3zyI70d6pkpnEVVtqvL8GxfLZaGNw7fuuM71c6EYVc3VDKqmyMp3rKrP49fHDt0PM1UQLhrpqOP7ERFJcm2olEyhydm1NU6AaB1yOWPBhgd4QrSSwitwXAD+/yKDvyQVHJ1ZCfCbIjIKY3QFe18+XAXEBg6deWBj2+6cjg4xtFcz1gg+VlLmUSQgVWDsXSx1Ii11tKz7UMPpWC8DvkYEBOgT06ifOOraq8L+Jq+nPDjUXYujuuApancot2EvcO+UdSNe59FgljsvDfGspChZOeYsFagIv1tOzhaQHr5DIP1vFn7ViVnm2PpXoOQtGffB5/3iGNUJLycwg8ImFyFc9POroZtvI0ncGFBl5WwggF8OaqWPmw07Bhl11+CdJb/LeTSk8xmPKfs3CFP1eOQW28ADn1FoHBoFCFFGjcJaEFxFghi7naMNT4R3wUWGUOwcP/wABMs= ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:30:26.967 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-11-16T13:30:28.587 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-11-16T13:30:28.587 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-11-16T13:30:29.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:28.115866+0000 mon.smithi151 (mon.0) 109 : cluster [DBG] mgrmap e17: smithi151.fcujra(active, since 6s) 2023-11-16T13:30:29.226 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:28.213627+0000 mon.smithi151 (mon.0) 110 : audit [INF] from='client.? 172.21.15.151:0/3169628981' entity='client.admin' 2023-11-16T13:30:30.362 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi167 2023-11-16T13:30:30.362 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-11-16T13:30:30.362 DEBUG:teuthology.orchestra.run.smithi167:> dd of=/etc/ceph/ceph.conf 2023-11-16T13:30:30.384 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-11-16T13:30:30.385 DEBUG:teuthology.orchestra.run.smithi167:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:30:30.446 INFO:tasks.cephadm:Adding host smithi167 to orchestrator... 2023-11-16T13:30:30.447 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch host add smithi167 2023-11-16T13:30:31.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:30.004433+0000 mon.smithi151 (mon.0) 111 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:31.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:30.683726+0000 mon.smithi151 (mon.0) 112 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:31.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:30.685408+0000 mon.smithi151 (mon.0) 113 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:30:31.226 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:30.686063+0000 mon.smithi151 (mon.0) 114 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:30:31.226 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:30.686867+0000 mgr.smithi151.fcujra (mgr.14164) 6 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(host_pattern='*'): Cannot place : No matching hosts 2023-11-16T13:30:31.226 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:30.912695+0000 mon.smithi151 (mon.0) 115 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:32.628 INFO:teuthology.orchestra.run.smithi151.stdout:Added host 'smithi167' with addr '172.21.15.167' 2023-11-16T13:30:32.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:31.688105+0000 mon.smithi151 (mon.0) 116 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:32.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:32.626439+0000 mon.smithi151 (mon.0) 117 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:33.032 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch host ls --format=json 2023-11-16T13:30:33.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:32.025883+0000 mgr.smithi151.fcujra (mgr.14164) 7 : audit [DBG] from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi167", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:30:33.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:32.626753+0000 mgr.smithi151.fcujra (mgr.14164) 8 : cephadm [INF] Added host smithi167 2023-11-16T13:30:34.737 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:30:34.738 INFO:teuthology.orchestra.run.smithi151.stdout:[{"addr": "172.21.15.151", "hostname": "smithi151", "labels": [], "status": ""}, {"addr": "172.21.15.167", "hostname": "smithi167", "labels": [], "status": ""}] 2023-11-16T13:30:35.058 INFO:tasks.cephadm:Setting crush tunables to default 2023-11-16T13:30:35.058 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd crush tunables default 2023-11-16T13:30:35.356 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:34.267351+0000 mon.smithi151 (mon.0) 118 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:35.356 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:34.612401+0000 mon.smithi151 (mon.0) 119 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:35.356 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:34.612909+0000 mon.smithi151 (mon.0) 120 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd/host:smithi151", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:30:35.356 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:34.615337+0000 mon.smithi151 (mon.0) 121 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:35.356 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:34.622035+0000 mgr.smithi151.fcujra (mgr.14164) 9 : cephadm [INF] Deploying daemon alertmanager.smithi151 on smithi151 2023-11-16T13:30:35.356 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:34.736235+0000 mgr.smithi151.fcujra (mgr.14164) 10 : audit [DBG] from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-11-16T13:30:36.853 INFO:teuthology.orchestra.run.smithi151.stderr:adjusted tunables profile to default 2023-11-16T13:30:37.119 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:36.317644+0000 mon.smithi151 (mon.0) 122 : audit [INF] from='client.? 172.21.15.151:0/1946592829' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-11-16T13:30:37.229 INFO:tasks.cephadm:Adding mon.smithi151 on smithi151 2023-11-16T13:30:37.229 INFO:tasks.cephadm:Adding mon.smithi167 on smithi167 2023-11-16T13:30:37.230 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch apply mon '2;smithi151:172.21.15.151=smithi151;smithi167:172.21.15.167=smithi167' 2023-11-16T13:30:38.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:36.849495+0000 mon.smithi151 (mon.0) 123 : audit [INF] from='client.? 172.21.15.151:0/1946592829' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-11-16T13:30:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:36.849535+0000 mon.smithi151 (mon.0) 124 : cluster [DBG] osdmap e5: 0 total, 0 up, 0 in 2023-11-16T13:30:38.529 INFO:teuthology.orchestra.run.smithi167.stdout:Scheduled mon update... 2023-11-16T13:30:38.909 DEBUG:teuthology.orchestra.run.smithi167:mon.smithi167> sudo journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi167.service 2023-11-16T13:30:38.912 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:30:38.912 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:30:38.940 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:-- Logs begin at Thu 2023-11-16 13:20:04 UTC. -- 2023-11-16T13:30:39.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:39 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:38.524102+0000 mgr.smithi151.fcujra (mgr.14164) 11 : audit [DBG] from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi151:172.21.15.151=smithi151;smithi167:172.21.15.167=smithi167", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:30:39.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:39 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:38.525476+0000 mgr.smithi151.fcujra (mgr.14164) 12 : cephadm [INF] Saving service mon spec with placement smithi151:172.21.15.151=smithi151;smithi167:172.21.15.167=smithi167;count:2 2023-11-16T13:30:39.986 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:39 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:38.528160+0000 mon.smithi151 (mon.0) 125 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:40.234 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:30:40.234 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:30:40.235 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:30:40.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:40.232551+0000 mon.smithi151 (mon.0) 126 : audit [DBG] from='client.? 172.21.15.167:0/141871440' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:30:41.606 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:30:41.607 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:30:42.935 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:30:42.935 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:30:42.936 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:30:43.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:42.099076+0000 mgr.smithi151.fcujra (mgr.14164) 13 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:30:43.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:42.933549+0000 mon.smithi151 (mon.0) 127 : audit [DBG] from='client.? 172.21.15.167:0/3089019789' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:30:44.307 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:30:44.307 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:30:45.445 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:44.099356+0000 mgr.smithi151.fcujra (mgr.14164) 14 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:30:45.680 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:30:45.680 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:30:45.682 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:30:46.917 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:45.611346+0000 mon.smithi151 (mon.0) 128 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:46.917 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:45.612663+0000 mon.smithi151 (mon.0) 129 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi151", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:30:46.917 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:45.614195+0000 mon.smithi151 (mon.0) 130 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi151", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2023-11-16T13:30:46.918 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:45.614732+0000 mon.smithi151 (mon.0) 131 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:30:46.918 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:45.615219+0000 mgr.smithi151.fcujra (mgr.14164) 15 : cephadm [INF] Deploying daemon crash.smithi151 on smithi151 2023-11-16T13:30:46.918 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:45.678648+0000 mon.smithi151 (mon.0) 132 : audit [DBG] from='client.? 172.21.15.167:0/302000389' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:30:47.047 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:30:47.048 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:30:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:46.099687+0000 mgr.smithi151.fcujra (mgr.14164) 16 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:30:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:47.077907+0000 mon.smithi151 (mon.0) 133 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:47.227487+0000 mon.smithi151 (mon.0) 134 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:47.348711+0000 mon.smithi151 (mon.0) 135 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:47.353783+0000 mon.smithi151 (mon.0) 136 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:47.354139+0000 mon.smithi151 (mon.0) 137 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-11-16T13:30:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:47.360059+0000 mon.smithi151 (mon.0) 138 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:30:48.440 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:30:48.440 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:30:48.442 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:30:49.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:47.354456+0000 mgr.smithi151.fcujra (mgr.14164) 17 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-11-16T13:30:49.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:30:47.363060+0000 mgr.smithi151.fcujra (mgr.14164) 18 : cephadm [INF] Deploying daemon grafana.smithi151 on smithi151 2023-11-16T13:30:49.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:48.438783+0000 mon.smithi151 (mon.0) 139 : audit [DBG] from='client.? 172.21.15.167:0/562358259' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:30:49.815 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:30:49.816 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:30:50.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:48.099964+0000 mgr.smithi151.fcujra (mgr.14164) 19 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:30:51.120 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:30:51.120 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:30:51.121 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:30:52.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:51 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:50.100283+0000 mgr.smithi151.fcujra (mgr.14164) 20 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:30:52.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:51 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:51.118523+0000 mon.smithi151 (mon.0) 140 : audit [DBG] from='client.? 172.21.15.167:0/968697912' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:30:52.508 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:30:52.508 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:30:53.997 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:30:53.997 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:30:53.999 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:30:54.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:52.100581+0000 mgr.smithi151.fcujra (mgr.14164) 21 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:30:55.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:54 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:53.995974+0000 mon.smithi151 (mon.0) 141 : audit [DBG] from='client.? 172.21.15.167:0/2504792009' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:30:55.392 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:30:55.393 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:30:56.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:55 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:54.100925+0000 mgr.smithi151.fcujra (mgr.14164) 22 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:30:56.740 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:30:56.740 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:30:56.742 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:30:57.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:56 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:56.739171+0000 mon.smithi151 (mon.0) 142 : audit [DBG] from='client.? 172.21.15.167:0/3191698785' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:30:58.116 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:30:58.116 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:30:58.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:56.101139+0000 mgr.smithi151.fcujra (mgr.14164) 23 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:30:59.452 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:30:59.452 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:30:59.453 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:31:00.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:30:58.101362+0000 mgr.smithi151.fcujra (mgr.14164) 24 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:00.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:30:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:30:59.450514+0000 mon.smithi151 (mon.0) 143 : audit [DBG] from='client.? 172.21.15.167:0/2976207413' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:00.784 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:31:00.784 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:31:02.146 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:31:02.146 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:31:02.148 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:31:02.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:00.101580+0000 mgr.smithi151.fcujra (mgr.14164) 25 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:03.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:02.144924+0000 mon.smithi151 (mon.0) 144 : audit [DBG] from='client.? 172.21.15.167:0/2745391050' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:03.535 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:31:03.535 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:31:04.041 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:02.101926+0000 mgr.smithi151.fcujra (mgr.14164) 26 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:04.042 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:03.770695+0000 mon.smithi151 (mon.0) 145 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:04.886 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:31:04.886 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:31:04.887 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:31:05.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:03.772957+0000 mgr.smithi151.fcujra (mgr.14164) 27 : cephadm [INF] Deploying daemon node-exporter.smithi151 on smithi151 2023-11-16T13:31:06.218 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:31:06.219 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:31:06.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:04.102182+0000 mgr.smithi151.fcujra (mgr.14164) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:06.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:04.884697+0000 mon.smithi151 (mon.0) 146 : audit [DBG] from='client.? 172.21.15.167:0/2834955835' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:07.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:06 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:06.102552+0000 mgr.smithi151.fcujra (mgr.14164) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:07.618 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:31:07.618 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:31:07.619 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:31:08.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:08 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:07.232270+0000 mon.smithi151 (mon.0) 147 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:08.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:08 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:07.615876+0000 mon.smithi151 (mon.0) 148 : audit [DBG] from='client.? 172.21.15.167:0/914921476' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:08.985 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:31:08.986 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:31:09.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:08.102824+0000 mgr.smithi151.fcujra (mgr.14164) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:10.378 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:31:10.378 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:31:10.379 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:31:11.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:10.103091+0000 mgr.smithi151.fcujra (mgr.14164) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:11.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:10.376286+0000 mon.smithi151 (mon.0) 149 : audit [DBG] from='client.? 172.21.15.167:0/1349961165' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:11.754 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:31:11.754 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:31:13.117 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:31:13.117 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:31:13.119 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:31:13.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:12.103304+0000 mgr.smithi151.fcujra (mgr.14164) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:13.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:12.721989+0000 mon.smithi151 (mon.0) 150 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:13.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:12.727519+0000 mgr.smithi151.fcujra (mgr.14164) 33 : cephadm [INF] Deploying daemon prometheus.smithi151 on smithi151 2023-11-16T13:31:13.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:13.115784+0000 mon.smithi151 (mon.0) 151 : audit [DBG] from='client.? 172.21.15.167:0/2414168414' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:14.485 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:31:14.485 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:31:15.844 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:31:15.844 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:31:15.846 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:31:15.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:14.103633+0000 mgr.smithi151.fcujra (mgr.14164) 34 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:16.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:15.842741+0000 mon.smithi151 (mon.0) 152 : audit [DBG] from='client.? 172.21.15.167:0/592677221' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:17.219 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:31:17.219 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:31:18.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:16.103958+0000 mgr.smithi151.fcujra (mgr.14164) 35 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:18.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:17.238101+0000 mon.smithi151 (mon.0) 153 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:18.574 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:31:18.574 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:31:18.576 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:31:19.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:18.573083+0000 mon.smithi151 (mon.0) 154 : audit [DBG] from='client.? 172.21.15.167:0/1585649015' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:19.955 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:31:19.956 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:31:20.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:18.104165+0000 mgr.smithi151.fcujra (mgr.14164) 36 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:21.326 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:31:21.326 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:31:21.327 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:31:22.115 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:20.104512+0000 mgr.smithi151.fcujra (mgr.14164) 37 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:22.116 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:21.324472+0000 mon.smithi151 (mon.0) 155 : audit [DBG] from='client.? 172.21.15.167:0/3341531920' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:22.705 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:31:22.706 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:31:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:22.239402+0000 mon.smithi151 (mon.0) 156 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:31:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:22.241030+0000 mon.smithi151 (mon.0) 157 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:31:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:22.598460+0000 mon.smithi151 (mon.0) 158 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:22.599295+0000 mon.smithi151 (mon.0) 159 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:31:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:22.600116+0000 mon.smithi151 (mon.0) 160 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:22.600714+0000 mon.smithi151 (mon.0) 161 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:31:24.204 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:31:24.204 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:31:24.206 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:31:24.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:22.104747+0000 mgr.smithi151.fcujra (mgr.14164) 38 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:24.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:23.089914+0000 mon.smithi151 (mon.0) 162 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:24.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:23.555863+0000 mon.smithi151 (mon.0) 163 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:24.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:24.202802+0000 mon.smithi151 (mon.0) 164 : audit [DBG] from='client.? 172.21.15.167:0/2419853850' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:24.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:24.798517+0000 mon.smithi151 (mon.0) 165 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:24.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:24.841090+0000 mon.smithi151 (mon.0) 166 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:25.647 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:31:25.647 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:31:26.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:25 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:24.105154+0000 mgr.smithi151.fcujra (mgr.14164) 39 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:26.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:25 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:24.798972+0000 mgr.smithi151.fcujra (mgr.14164) 40 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-11-16T13:31:26.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:25 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:24.813129+0000 mgr.smithi151.fcujra (mgr.14164) 41 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:31:27.007 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:31:27.007 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:31:27.009 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:31:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:26.027003+0000 mon.smithi151 (mon.0) 167 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:26.105504+0000 mgr.smithi151.fcujra (mgr.14164) 42 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:26.351047+0000 mon.smithi151 (mon.0) 168 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:26.351853+0000 mon.smithi151 (mon.0) 169 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd/host:smithi167", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:31:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:26.355052+0000 mon.smithi151 (mon.0) 170 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:26.356961+0000 mon.smithi151 (mon.0) 171 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi167", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:31:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:26.359852+0000 mon.smithi151 (mon.0) 172 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi167", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2023-11-16T13:31:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:26.360588+0000 mon.smithi151 (mon.0) 173 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:26.361197+0000 mgr.smithi151.fcujra (mgr.14164) 43 : cephadm [INF] Deploying daemon crash.smithi167 on smithi167 2023-11-16T13:31:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:27.006158+0000 mon.smithi151 (mon.0) 174 : audit [DBG] from='client.? 172.21.15.167:0/4120167091' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:28.513 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:31:28.513 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:31:28.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:27.241011+0000 mon.smithi151 (mon.0) 175 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:28.105891+0000 mgr.smithi151.fcujra (mgr.14164) 44 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:28.407562+0000 mon.smithi151 (mon.0) 176 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:28.409680+0000 mon.smithi151 (mon.0) 177 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi167.xxnzsz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:31:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:28.411722+0000 mon.smithi151 (mon.0) 178 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi167.xxnzsz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-11-16T13:31:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:28.412415+0000 mon.smithi151 (mon.0) 179 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:31:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:28.413188+0000 mon.smithi151 (mon.0) 180 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:28.413736+0000 mgr.smithi151.fcujra (mgr.14164) 45 : cephadm [INF] Deploying daemon mgr.smithi167.xxnzsz on smithi167 2023-11-16T13:31:30.293 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:31:30.294 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":1,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:29:31.114936Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-16T13:31:30.295 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 1 2023-11-16T13:31:31.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:30.106233+0000 mgr.smithi151.fcujra (mgr.14164) 46 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:31.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:30.213786+0000 mon.smithi151 (mon.0) 181 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:31.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:30.215845+0000 mon.smithi151 (mon.0) 182 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:31:31.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:30.216637+0000 mon.smithi151 (mon.0) 183 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:31.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:30.217262+0000 mgr.smithi151.fcujra (mgr.14164) 47 : cephadm [INF] Deploying daemon mon.smithi167 on smithi167 2023-11-16T13:31:31.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:30.292082+0000 mon.smithi151 (mon.0) 184 : audit [DBG] from='client.? 172.21.15.167:0/745140466' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:31.626 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-16T13:31:31.626 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-16T13:31:33.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: debug 2023-11-16T13:31:32.776+0000 7f71f2520700 1 mon.smithi167@-1(synchronizing).paxosservice(auth 1..7) refresh upgraded, format 0 -> 3 2023-11-16T13:31:38.188 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:32.106589+0000 mgr.smithi151.fcujra (mgr.14164) 48 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:38.188 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:32.784886+0000 mon.smithi151 (mon.0) 187 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:31:38.188 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:32.785099+0000 mon.smithi151 (mon.0) 188 : cluster [INF] mon.smithi151 calling monitor election 2023-11-16T13:31:38.188 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:32.797877+0000 mon.smithi151 (mon.0) 189 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:38.188 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:33.781159+0000 mon.smithi151 (mon.0) 190 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:38.188 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:34.106886+0000 mgr.smithi151.fcujra (mgr.14164) 49 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:38.188 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:34.781321+0000 mon.smithi151 (mon.0) 191 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:38.188 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:34.782899+0000 mon.smithi167 (mon.1) 1 : cluster [INF] mon.smithi167 calling monitor election 2023-11-16T13:31:38.188 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:35.329947+0000 mon.smithi151 (mon.0) 192 : audit [DBG] from='mgr.? 172.21.15.167:0/2228666649' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/crt"}]: dispatch 2023-11-16T13:31:38.188 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:35.781407+0000 mon.smithi151 (mon.0) 193 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:38.188 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:36.107185+0000 mgr.smithi151.fcujra (mgr.14164) 50 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:38.188 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:36.781505+0000 mon.smithi151 (mon.0) 194 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:38.189 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:37.781770+0000 mon.smithi151 (mon.0) 195 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:38.189 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:37.812163+0000 mon.smithi151 (mon.0) 196 : cluster [INF] mon.smithi151 is new leader, mons smithi151,smithi167 in quorum (ranks 0,1) 2023-11-16T13:31:38.189 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:37.820693+0000 mon.smithi151 (mon.0) 197 : cluster [DBG] monmap e2: 2 mons at {smithi151=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],smithi167=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0]} 2023-11-16T13:31:38.189 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:37.820754+0000 mon.smithi151 (mon.0) 198 : cluster [DBG] fsmap 2023-11-16T13:31:38.189 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:37.820793+0000 mon.smithi151 (mon.0) 199 : cluster [DBG] osdmap e5: 0 total, 0 up, 0 in 2023-11-16T13:31:38.189 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:37.821056+0000 mon.smithi151 (mon.0) 200 : cluster [DBG] mgrmap e17: smithi151.fcujra(active, since 75s) 2023-11-16T13:31:38.189 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:37.822322+0000 mon.smithi151 (mon.0) 201 : cluster [DBG] Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:31:38.189 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:37.823423+0000 mon.smithi151 (mon.0) 202 : audit [DBG] from='mgr.? 172.21.15.167:0/2228666649' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-16T13:31:38.189 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:37.824722+0000 mon.smithi151 (mon.0) 203 : audit [DBG] from='mgr.? 172.21.15.167:0/2228666649' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/key"}]: dispatch 2023-11-16T13:31:38.189 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:37.825188+0000 mon.smithi151 (mon.0) 204 : audit [DBG] from='mgr.? 172.21.15.167:0/2228666649' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:31:38.189 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:37.828336+0000 mon.smithi151 (mon.0) 205 : cluster [INF] overall HEALTH_OK 2023-11-16T13:31:38.189 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:37.830345+0000 mon.smithi151 (mon.0) 206 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:32.106589+0000 mgr.smithi151.fcujra (mgr.14164) 48 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:32.784886+0000 mon.smithi151 (mon.0) 187 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:31:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:32.785099+0000 mon.smithi151 (mon.0) 188 : cluster [INF] mon.smithi151 calling monitor election 2023-11-16T13:31:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:32.797877+0000 mon.smithi151 (mon.0) 189 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:33.781159+0000 mon.smithi151 (mon.0) 190 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:34.106886+0000 mgr.smithi151.fcujra (mgr.14164) 49 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:34.781321+0000 mon.smithi151 (mon.0) 191 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:34.782899+0000 mon.smithi167 (mon.1) 1 : cluster [INF] mon.smithi167 calling monitor election 2023-11-16T13:31:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:35.329947+0000 mon.smithi151 (mon.0) 192 : audit [DBG] from='mgr.? 172.21.15.167:0/2228666649' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/crt"}]: dispatch 2023-11-16T13:31:38.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:35.781407+0000 mon.smithi151 (mon.0) 193 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:38.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:36.107185+0000 mgr.smithi151.fcujra (mgr.14164) 50 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:38.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:36.781505+0000 mon.smithi151 (mon.0) 194 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:38.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:37.781770+0000 mon.smithi151 (mon.0) 195 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:38.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:37.812163+0000 mon.smithi151 (mon.0) 196 : cluster [INF] mon.smithi151 is new leader, mons smithi151,smithi167 in quorum (ranks 0,1) 2023-11-16T13:31:38.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:37.820693+0000 mon.smithi151 (mon.0) 197 : cluster [DBG] monmap e2: 2 mons at {smithi151=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],smithi167=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0]} 2023-11-16T13:31:38.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:37.820754+0000 mon.smithi151 (mon.0) 198 : cluster [DBG] fsmap 2023-11-16T13:31:38.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:37.820793+0000 mon.smithi151 (mon.0) 199 : cluster [DBG] osdmap e5: 0 total, 0 up, 0 in 2023-11-16T13:31:38.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:37.821056+0000 mon.smithi151 (mon.0) 200 : cluster [DBG] mgrmap e17: smithi151.fcujra(active, since 75s) 2023-11-16T13:31:38.333 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:37.822322+0000 mon.smithi151 (mon.0) 201 : cluster [DBG] Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:31:38.333 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:37.823423+0000 mon.smithi151 (mon.0) 202 : audit [DBG] from='mgr.? 172.21.15.167:0/2228666649' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-16T13:31:38.333 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:37.824722+0000 mon.smithi151 (mon.0) 203 : audit [DBG] from='mgr.? 172.21.15.167:0/2228666649' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/key"}]: dispatch 2023-11-16T13:31:38.333 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:37.825188+0000 mon.smithi151 (mon.0) 204 : audit [DBG] from='mgr.? 172.21.15.167:0/2228666649' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:31:38.333 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:37.828336+0000 mon.smithi151 (mon.0) 205 : cluster [INF] overall HEALTH_OK 2023-11-16T13:31:38.333 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:37.830345+0000 mon.smithi151 (mon.0) 206 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:39.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:37.832041+0000 mgr.smithi151.fcujra (mgr.14164) 51 : cephadm [INF] Deploying daemon node-exporter.smithi167 on smithi167 2023-11-16T13:31:39.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:37.881756+0000 mon.smithi151 (mon.0) 207 : cluster [DBG] mgrmap e18: smithi151.fcujra(active, since 75s), standbys: smithi167.xxnzsz 2023-11-16T13:31:39.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:37.881917+0000 mon.smithi151 (mon.0) 208 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi167.xxnzsz", "id": "smithi167.xxnzsz"}]: dispatch 2023-11-16T13:31:39.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:38.782011+0000 mon.smithi151 (mon.0) 209 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:39.256 INFO:teuthology.orchestra.run.smithi167.stdout: 2023-11-16T13:31:39.256 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":2,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","modified":"2023-11-16T13:31:32.782066Z","created":"2023-11-16T13:29:31.114936Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi151","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi167","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:3300","nonce":0},{"type":"v1","addr":"172.21.15.167:6789","nonce":0}]},"addr":"172.21.15.167:6789/0","public_addr":"172.21.15.167:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-11-16T13:31:39.257 INFO:teuthology.orchestra.run.smithi167.stderr:dumped monmap epoch 2 2023-11-16T13:31:39.279 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:31:37.832041+0000 mgr.smithi151.fcujra (mgr.14164) 51 : cephadm [INF] Deploying daemon node-exporter.smithi167 on smithi167 2023-11-16T13:31:39.280 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:37.881756+0000 mon.smithi151 (mon.0) 207 : cluster [DBG] mgrmap e18: smithi151.fcujra(active, since 75s), standbys: smithi167.xxnzsz 2023-11-16T13:31:39.280 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:37.881917+0000 mon.smithi151 (mon.0) 208 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi167.xxnzsz", "id": "smithi167.xxnzsz"}]: dispatch 2023-11-16T13:31:39.280 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:38.782011+0000 mon.smithi151 (mon.0) 209 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:31:39.576 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-11-16T13:31:39.576 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph config generate-minimal-conf 2023-11-16T13:31:40.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:39 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:38.107541+0000 mgr.smithi151.fcujra (mgr.14164) 52 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:40.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:39 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:39.253287+0000 mon.smithi151 (mon.0) 210 : audit [DBG] from='client.? 172.21.15.167:0/1556545453' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:40.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:39 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:38.107541+0000 mgr.smithi151.fcujra (mgr.14164) 52 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:40.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:39 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:39.253287+0000 mon.smithi151 (mon.0) 210 : audit [DBG] from='client.? 172.21.15.167:0/1556545453' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-16T13:31:40.938 INFO:teuthology.orchestra.run.smithi151.stdout:# minimal ceph.conf for 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:31:40.938 INFO:teuthology.orchestra.run.smithi151.stdout:[global] 2023-11-16T13:31:40.938 INFO:teuthology.orchestra.run.smithi151.stdout: fsid = 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:31:40.938 INFO:teuthology.orchestra.run.smithi151.stdout: mon_host = [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] [v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0] 2023-11-16T13:31:41.301 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-11-16T13:31:41.301 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-11-16T13:31:41.301 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/ceph/ceph.conf 2023-11-16T13:31:41.331 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-11-16T13:31:41.331 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:31:41.398 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-11-16T13:31:41.399 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/ceph/ceph.conf 2023-11-16T13:31:41.437 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-11-16T13:31:41.437 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:31:41.508 INFO:tasks.cephadm:Deploying OSDs... 2023-11-16T13:31:41.508 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-11-16T13:31:41.508 DEBUG:teuthology.orchestra.run.smithi151:> dd if=/scratch_devs of=/dev/stdout 2023-11-16T13:31:41.526 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-11-16T13:31:41.527 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_1 2023-11-16T13:31:41.584 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-11-16T13:31:41.585 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-16T13:31:41.585 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 6h/6d Inode: 392706 Links: 1 2023-11-16T13:31:41.585 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-16T13:31:41.585 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2023-11-16T13:31:41.585 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2023-11-16 13:31:22.165921285 +0000 2023-11-16T13:31:41.585 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2023-11-16 13:27:29.123231800 +0000 2023-11-16T13:31:41.585 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2023-11-16 13:27:29.123231800 +0000 2023-11-16T13:31:41.585 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: - 2023-11-16T13:31:41.585 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-11-16T13:31:41.650 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2023-11-16T13:31:41.650 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2023-11-16T13:31:41.650 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000203296 s, 2.5 MB/s 2023-11-16T13:31:41.651 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-11-16T13:31:41.710 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_2 2023-11-16T13:31:41.768 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-11-16T13:31:41.768 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-16T13:31:41.768 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 6h/6d Inode: 394689 Links: 1 2023-11-16T13:31:41.768 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-16T13:31:41.768 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2023-11-16T13:31:41.769 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2023-11-16 13:31:22.165921285 +0000 2023-11-16T13:31:41.769 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2023-11-16 13:27:29.652225406 +0000 2023-11-16T13:31:41.769 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2023-11-16 13:27:29.652225406 +0000 2023-11-16T13:31:41.769 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: - 2023-11-16T13:31:41.769 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-11-16T13:31:41.834 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2023-11-16T13:31:41.834 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2023-11-16T13:31:41.834 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000195894 s, 2.6 MB/s 2023-11-16T13:31:41.835 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-11-16T13:31:41.894 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_3 2023-11-16T13:31:41.952 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-11-16T13:31:41.952 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-16T13:31:41.952 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 6h/6d Inode: 394776 Links: 1 2023-11-16T13:31:41.952 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-16T13:31:41.952 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2023-11-16T13:31:41.953 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2023-11-16 13:31:22.165921285 +0000 2023-11-16T13:31:41.953 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2023-11-16 13:27:30.205218722 +0000 2023-11-16T13:31:41.953 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2023-11-16 13:27:30.205218722 +0000 2023-11-16T13:31:41.953 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: - 2023-11-16T13:31:41.953 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-11-16T13:31:41.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:41 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:40.107855+0000 mgr.smithi151.fcujra (mgr.14164) 53 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:41.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:41 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:40.937048+0000 mon.smithi151 (mon.0) 211 : audit [DBG] from='client.? 172.21.15.151:0/3174609525' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:42.008 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2023-11-16T13:31:42.008 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2023-11-16T13:31:42.008 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000217168 s, 2.4 MB/s 2023-11-16T13:31:42.009 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-11-16T13:31:42.068 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_4 2023-11-16T13:31:42.126 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-11-16T13:31:42.126 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-16T13:31:42.126 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 6h/6d Inode: 399513 Links: 1 2023-11-16T13:31:42.126 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-16T13:31:42.126 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2023-11-16T13:31:42.127 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2023-11-16 13:31:22.165921285 +0000 2023-11-16T13:31:42.127 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2023-11-16 13:27:30.717212534 +0000 2023-11-16T13:31:42.127 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2023-11-16 13:27:30.717212534 +0000 2023-11-16T13:31:42.127 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: - 2023-11-16T13:31:42.127 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-11-16T13:31:42.192 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2023-11-16T13:31:42.192 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2023-11-16T13:31:42.192 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000233607 s, 2.2 MB/s 2023-11-16T13:31:42.193 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-11-16T13:31:42.252 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2023-11-16T13:31:42.253 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/scratch_devs of=/dev/stdout 2023-11-16T13:31:42.278 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-11-16T13:31:42.278 DEBUG:teuthology.orchestra.run.smithi167:> stat /dev/vg_nvme/lv_1 2023-11-16T13:31:42.340 INFO:teuthology.orchestra.run.smithi167.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-11-16T13:31:42.340 INFO:teuthology.orchestra.run.smithi167.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-16T13:31:42.340 INFO:teuthology.orchestra.run.smithi167.stdout:Device: 6h/6d Inode: 390760 Links: 1 2023-11-16T13:31:42.341 INFO:teuthology.orchestra.run.smithi167.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-16T13:31:42.341 INFO:teuthology.orchestra.run.smithi167.stdout:Context: system_u:object_r:device_t:s0 2023-11-16T13:31:42.341 INFO:teuthology.orchestra.run.smithi167.stdout:Access: 2023-11-16 13:31:38.238686479 +0000 2023-11-16T13:31:42.341 INFO:teuthology.orchestra.run.smithi167.stdout:Modify: 2023-11-16 13:27:24.710367052 +0000 2023-11-16T13:31:42.341 INFO:teuthology.orchestra.run.smithi167.stdout:Change: 2023-11-16 13:27:24.710367052 +0000 2023-11-16T13:31:42.341 INFO:teuthology.orchestra.run.smithi167.stdout: Birth: - 2023-11-16T13:31:42.341 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-11-16T13:31:42.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:41 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:40.107855+0000 mgr.smithi151.fcujra (mgr.14164) 53 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:42.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:41 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:40.937048+0000 mon.smithi151 (mon.0) 211 : audit [DBG] from='client.? 172.21.15.151:0/3174609525' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:42.374 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records in 2023-11-16T13:31:42.374 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records out 2023-11-16T13:31:42.375 INFO:teuthology.orchestra.run.smithi167.stderr:512 bytes copied, 0.000234276 s, 2.2 MB/s 2023-11-16T13:31:42.375 DEBUG:teuthology.orchestra.run.smithi167:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-11-16T13:31:42.436 DEBUG:teuthology.orchestra.run.smithi167:> stat /dev/vg_nvme/lv_2 2023-11-16T13:31:42.496 INFO:teuthology.orchestra.run.smithi167.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-11-16T13:31:42.496 INFO:teuthology.orchestra.run.smithi167.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-16T13:31:42.496 INFO:teuthology.orchestra.run.smithi167.stdout:Device: 6h/6d Inode: 397551 Links: 1 2023-11-16T13:31:42.496 INFO:teuthology.orchestra.run.smithi167.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-16T13:31:42.496 INFO:teuthology.orchestra.run.smithi167.stdout:Context: system_u:object_r:device_t:s0 2023-11-16T13:31:42.496 INFO:teuthology.orchestra.run.smithi167.stdout:Access: 2023-11-16 13:31:38.238686479 +0000 2023-11-16T13:31:42.496 INFO:teuthology.orchestra.run.smithi167.stdout:Modify: 2023-11-16 13:27:25.217357930 +0000 2023-11-16T13:31:42.496 INFO:teuthology.orchestra.run.smithi167.stdout:Change: 2023-11-16 13:27:25.217357930 +0000 2023-11-16T13:31:42.497 INFO:teuthology.orchestra.run.smithi167.stdout: Birth: - 2023-11-16T13:31:42.497 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-11-16T13:31:42.563 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records in 2023-11-16T13:31:42.563 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records out 2023-11-16T13:31:42.563 INFO:teuthology.orchestra.run.smithi167.stderr:512 bytes copied, 0.000219007 s, 2.3 MB/s 2023-11-16T13:31:42.565 DEBUG:teuthology.orchestra.run.smithi167:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-11-16T13:31:42.625 DEBUG:teuthology.orchestra.run.smithi167:> stat /dev/vg_nvme/lv_3 2023-11-16T13:31:42.684 INFO:teuthology.orchestra.run.smithi167.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-11-16T13:31:42.684 INFO:teuthology.orchestra.run.smithi167.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-16T13:31:42.684 INFO:teuthology.orchestra.run.smithi167.stdout:Device: 6h/6d Inode: 398534 Links: 1 2023-11-16T13:31:42.684 INFO:teuthology.orchestra.run.smithi167.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-16T13:31:42.684 INFO:teuthology.orchestra.run.smithi167.stdout:Context: system_u:object_r:device_t:s0 2023-11-16T13:31:42.684 INFO:teuthology.orchestra.run.smithi167.stdout:Access: 2023-11-16 13:31:38.238686479 +0000 2023-11-16T13:31:42.684 INFO:teuthology.orchestra.run.smithi167.stdout:Modify: 2023-11-16 13:27:25.737348574 +0000 2023-11-16T13:31:42.684 INFO:teuthology.orchestra.run.smithi167.stdout:Change: 2023-11-16 13:27:25.737348574 +0000 2023-11-16T13:31:42.685 INFO:teuthology.orchestra.run.smithi167.stdout: Birth: - 2023-11-16T13:31:42.685 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-11-16T13:31:42.751 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records in 2023-11-16T13:31:42.751 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records out 2023-11-16T13:31:42.751 INFO:teuthology.orchestra.run.smithi167.stderr:512 bytes copied, 0.000210768 s, 2.4 MB/s 2023-11-16T13:31:42.752 DEBUG:teuthology.orchestra.run.smithi167:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-11-16T13:31:42.811 DEBUG:teuthology.orchestra.run.smithi167:> stat /dev/vg_nvme/lv_4 2023-11-16T13:31:42.869 INFO:teuthology.orchestra.run.smithi167.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-11-16T13:31:42.870 INFO:teuthology.orchestra.run.smithi167.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-16T13:31:42.870 INFO:teuthology.orchestra.run.smithi167.stdout:Device: 6h/6d Inode: 398579 Links: 1 2023-11-16T13:31:42.870 INFO:teuthology.orchestra.run.smithi167.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-16T13:31:42.870 INFO:teuthology.orchestra.run.smithi167.stdout:Context: system_u:object_r:device_t:s0 2023-11-16T13:31:42.870 INFO:teuthology.orchestra.run.smithi167.stdout:Access: 2023-11-16 13:31:38.238686479 +0000 2023-11-16T13:31:42.870 INFO:teuthology.orchestra.run.smithi167.stdout:Modify: 2023-11-16 13:27:26.258339200 +0000 2023-11-16T13:31:42.870 INFO:teuthology.orchestra.run.smithi167.stdout:Change: 2023-11-16 13:27:26.258339200 +0000 2023-11-16T13:31:42.870 INFO:teuthology.orchestra.run.smithi167.stdout: Birth: - 2023-11-16T13:31:42.871 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-11-16T13:31:42.936 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records in 2023-11-16T13:31:42.937 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records out 2023-11-16T13:31:42.937 INFO:teuthology.orchestra.run.smithi167.stderr:512 bytes copied, 0.000241314 s, 2.1 MB/s 2023-11-16T13:31:42.938 DEBUG:teuthology.orchestra.run.smithi167:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-11-16T13:31:42.995 INFO:tasks.cephadm:Deploying osd.0 on smithi151 with /dev/vg_nvme/lv_4... 2023-11-16T13:31:42.995 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-11-16T13:31:43.303 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:42.108057+0000 mgr.smithi151.fcujra (mgr.14164) 54 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:43.303 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:42.249890+0000 mon.smithi151 (mon.0) 212 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:42.108057+0000 mgr.smithi151.fcujra (mgr.14164) 54 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:42.249890+0000 mon.smithi151 (mon.0) 212 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:44.422 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-11-16T13:31:44.422 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-11-16T13:31:44.422 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2023-11-16T13:31:44.422 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2023-11-16T13:31:44.422 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111159 s, 943 MB/s 2023-11-16T13:31:44.422 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2023-11-16T13:31:44.926 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch daemon add osd smithi151:vg_nvme/lv_4 2023-11-16T13:31:45.555 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:44.108327+0000 mgr.smithi151.fcujra (mgr.14164) 55 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:44.108327+0000 mgr.smithi151.fcujra (mgr.14164) 55 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:47.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:46.234322+0000 mon.smithi151 (mon.0) 213 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:31:47.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:46.235803+0000 mon.smithi151 (mon.0) 214 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:31:47.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:46.236317+0000 mon.smithi151 (mon.0) 215 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:47.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:46.761391+0000 mon.smithi151 (mon.0) 216 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:47.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:46.762486+0000 mon.smithi151 (mon.0) 217 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:31:47.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:46.763280+0000 mon.smithi151 (mon.0) 218 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:47.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:46.763791+0000 mon.smithi151 (mon.0) 219 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:31:47.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:46.234322+0000 mon.smithi151 (mon.0) 213 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:31:47.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:46.235803+0000 mon.smithi151 (mon.0) 214 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:31:47.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:46.236317+0000 mon.smithi151 (mon.0) 215 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:47.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:46.761391+0000 mon.smithi151 (mon.0) 216 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:47.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:46.762486+0000 mon.smithi151 (mon.0) 217 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:31:47.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:46.763280+0000 mon.smithi151 (mon.0) 218 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:47.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:46.763791+0000 mon.smithi151 (mon.0) 219 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:31:48.220 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:46.108591+0000 mgr.smithi151.fcujra (mgr.14164) 56 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:48.220 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:46.233344+0000 mgr.smithi151.fcujra (mgr.14164) 57 : audit [DBG] from='client.14242 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:31:48.220 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:47.253649+0000 mon.smithi151 (mon.0) 220 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:48.220 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:47.748604+0000 mon.smithi151 (mon.0) 221 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:48.220 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:47.797366+0000 mon.smithi151 (mon.0) 222 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:48.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:46.108591+0000 mgr.smithi151.fcujra (mgr.14164) 56 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:48.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:46.233344+0000 mgr.smithi151.fcujra (mgr.14164) 57 : audit [DBG] from='client.14242 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:31:48.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:47.253649+0000 mon.smithi151 (mon.0) 220 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:48.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:47.748604+0000 mon.smithi151 (mon.0) 221 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:48.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:47.797366+0000 mon.smithi151 (mon.0) 222 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:49.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:47.749110+0000 mgr.smithi151.fcujra (mgr.14164) 58 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.conf 2023-11-16T13:31:49.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:47.768282+0000 mgr.smithi151.fcujra (mgr.14164) 59 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:31:49.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:48.052728+0000 mon.smithi151 (mon.0) 223 : audit [INF] from='client.? 172.21.15.151:0/1639436238' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d0acfa3f-92df-4f58-8a5e-3578a88f486c"}]: dispatch 2023-11-16T13:31:49.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:48.057927+0000 mon.smithi151 (mon.0) 224 : audit [INF] from='client.? 172.21.15.151:0/1639436238' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d0acfa3f-92df-4f58-8a5e-3578a88f486c"}]': finished 2023-11-16T13:31:49.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:48.058002+0000 mon.smithi151 (mon.0) 225 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-11-16T13:31:49.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:48.058122+0000 mon.smithi151 (mon.0) 226 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:31:49.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:48.503223+0000 mon.smithi151 (mon.0) 227 : audit [DBG] from='client.? 172.21.15.151:0/3449934388' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:31:49.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:31:47.749110+0000 mgr.smithi151.fcujra (mgr.14164) 58 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.conf 2023-11-16T13:31:49.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:31:47.768282+0000 mgr.smithi151.fcujra (mgr.14164) 59 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:31:49.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:48.052728+0000 mon.smithi151 (mon.0) 223 : audit [INF] from='client.? 172.21.15.151:0/1639436238' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d0acfa3f-92df-4f58-8a5e-3578a88f486c"}]: dispatch 2023-11-16T13:31:49.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:48.057927+0000 mon.smithi151 (mon.0) 224 : audit [INF] from='client.? 172.21.15.151:0/1639436238' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d0acfa3f-92df-4f58-8a5e-3578a88f486c"}]': finished 2023-11-16T13:31:49.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:48.058002+0000 mon.smithi151 (mon.0) 225 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-11-16T13:31:49.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:48.058122+0000 mon.smithi151 (mon.0) 226 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:31:49.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:48.503223+0000 mon.smithi151 (mon.0) 227 : audit [DBG] from='client.? 172.21.15.151:0/3449934388' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:31:50.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:48.108972+0000 mgr.smithi151.fcujra (mgr.14164) 60 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:50.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:49.099878+0000 mon.smithi151 (mon.0) 228 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:50.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:49.357353+0000 mon.smithi151 (mon.0) 229 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:50.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:49.362550+0000 mon.smithi151 (mon.0) 230 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:50.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:49.365810+0000 mon.smithi151 (mon.0) 231 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:31:50.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:49.366344+0000 mon.smithi151 (mon.0) 232 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-11-16T13:31:50.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:49.366803+0000 mon.smithi151 (mon.0) 233 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:48.108972+0000 mgr.smithi151.fcujra (mgr.14164) 60 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:49.099878+0000 mon.smithi151 (mon.0) 228 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:49.357353+0000 mon.smithi151 (mon.0) 229 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:49.362550+0000 mon.smithi151 (mon.0) 230 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:49.365810+0000 mon.smithi151 (mon.0) 231 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:31:50.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:49.366344+0000 mon.smithi151 (mon.0) 232 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-11-16T13:31:50.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:49.366803+0000 mon.smithi151 (mon.0) 233 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:51.138 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:49.331212+0000 mgr.smithi151.fcujra (mgr.14164) 61 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-11-16T13:31:51.138 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:49.365568+0000 mgr.smithi151.fcujra (mgr.14164) 62 : cephadm [INF] Reconfiguring mon.smithi151 (unknown last config time)... 2023-11-16T13:31:51.139 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:49.367250+0000 mgr.smithi151.fcujra (mgr.14164) 63 : cephadm [INF] Reconfiguring daemon mon.smithi151 on smithi151 2023-11-16T13:31:51.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:31:49.331212+0000 mgr.smithi151.fcujra (mgr.14164) 61 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-11-16T13:31:51.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:31:49.365568+0000 mgr.smithi151.fcujra (mgr.14164) 62 : cephadm [INF] Reconfiguring mon.smithi151 (unknown last config time)... 2023-11-16T13:31:51.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:31:49.367250+0000 mgr.smithi151.fcujra (mgr.14164) 63 : cephadm [INF] Reconfiguring daemon mon.smithi151 on smithi151 2023-11-16T13:31:52.138 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:51 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:50.109276+0000 mgr.smithi151.fcujra (mgr.14164) 64 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:52.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:51 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:50.109276+0000 mgr.smithi151.fcujra (mgr.14164) 64 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:53.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:52.109541+0000 mgr.smithi151.fcujra (mgr.14164) 65 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:53.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:52.379921+0000 mon.smithi151 (mon.0) 234 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:53.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:52.380396+0000 mgr.smithi151.fcujra (mgr.14164) 66 : cephadm [INF] Reconfiguring mgr.smithi151.fcujra (unknown last config time)... 2023-11-16T13:31:53.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:52.380650+0000 mon.smithi151 (mon.0) 235 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi151.fcujra", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:31:53.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:52.381268+0000 mon.smithi151 (mon.0) 236 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:31:53.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:52.381766+0000 mon.smithi151 (mon.0) 237 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:53.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:52.382203+0000 mgr.smithi151.fcujra (mgr.14164) 67 : cephadm [INF] Reconfiguring daemon mgr.smithi151.fcujra on smithi151 2023-11-16T13:31:53.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:53.223518+0000 mon.smithi151 (mon.0) 238 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-11-16T13:31:53.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:53.224190+0000 mon.smithi151 (mon.0) 239 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:53.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:52.109541+0000 mgr.smithi151.fcujra (mgr.14164) 65 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:53.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:52.379921+0000 mon.smithi151 (mon.0) 234 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:53.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:31:52.380396+0000 mgr.smithi151.fcujra (mgr.14164) 66 : cephadm [INF] Reconfiguring mgr.smithi151.fcujra (unknown last config time)... 2023-11-16T13:31:53.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:52.380650+0000 mon.smithi151 (mon.0) 235 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi151.fcujra", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:31:53.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:52.381268+0000 mon.smithi151 (mon.0) 236 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:31:53.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:52.381766+0000 mon.smithi151 (mon.0) 237 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:53.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:31:52.382203+0000 mgr.smithi151.fcujra (mgr.14164) 67 : cephadm [INF] Reconfiguring daemon mgr.smithi151.fcujra on smithi151 2023-11-16T13:31:53.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:53.223518+0000 mon.smithi151 (mon.0) 238 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-11-16T13:31:53.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:53.224190+0000 mon.smithi151 (mon.0) 239 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:54.633 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:54 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:53.224688+0000 mgr.smithi151.fcujra (mgr.14164) 68 : cephadm [INF] Deploying daemon osd.0 on smithi151 2023-11-16T13:31:54.633 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:54 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:53.904623+0000 mon.smithi151 (mon.0) 240 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:54.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:54 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:31:53.224688+0000 mgr.smithi151.fcujra (mgr.14164) 68 : cephadm [INF] Deploying daemon osd.0 on smithi151 2023-11-16T13:31:54.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:54 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:53.904623+0000 mon.smithi151 (mon.0) 240 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:55 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:53.905157+0000 mgr.smithi151.fcujra (mgr.14164) 69 : cephadm [INF] Reconfiguring alertmanager.smithi151 (dependencies changed)... 2023-11-16T13:31:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:55 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:53.907331+0000 mgr.smithi151.fcujra (mgr.14164) 70 : cephadm [INF] Reconfiguring daemon alertmanager.smithi151 on smithi151 2023-11-16T13:31:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:55 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:54.109855+0000 mgr.smithi151.fcujra (mgr.14164) 71 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:55.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:55 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:31:53.905157+0000 mgr.smithi151.fcujra (mgr.14164) 69 : cephadm [INF] Reconfiguring alertmanager.smithi151 (dependencies changed)... 2023-11-16T13:31:55.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:55 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:31:53.907331+0000 mgr.smithi151.fcujra (mgr.14164) 70 : cephadm [INF] Reconfiguring daemon alertmanager.smithi151 on smithi151 2023-11-16T13:31:55.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:55 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:54.109855+0000 mgr.smithi151.fcujra (mgr.14164) 71 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:56.084 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 0 on host 'smithi151' 2023-11-16T13:31:56.553 DEBUG:teuthology.orchestra.run.smithi151:osd.0> sudo journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.0.service 2023-11-16T13:31:56.556 INFO:tasks.cephadm:Deploying osd.1 on smithi151 with /dev/vg_nvme/lv_3... 2023-11-16T13:31:56.556 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-11-16T13:31:56.588 INFO:journalctl@ceph.osd.0.smithi151.stdout:-- Logs begin at Thu 2023-11-16 13:20:07 UTC. -- 2023-11-16T13:31:57.192 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:56.077354+0000 mon.smithi151 (mon.0) 241 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:57.193 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:56.080840+0000 mon.smithi151 (mon.0) 242 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:57.193 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:56.110097+0000 mgr.smithi151.fcujra (mgr.14164) 72 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:57.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:56.077354+0000 mon.smithi151 (mon.0) 241 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:57.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:56.080840+0000 mon.smithi151 (mon.0) 242 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:57.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:56.110097+0000 mgr.smithi151.fcujra (mgr.14164) 72 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:58.541 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-11-16T13:31:58.542 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-11-16T13:31:58.542 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2023-11-16T13:31:58.542 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2023-11-16T13:31:58.542 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0130698 s, 802 MB/s 2023-11-16T13:31:58.542 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2023-11-16T13:31:58.993 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch daemon add osd smithi151:vg_nvme/lv_3 2023-11-16T13:31:59.427 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:31:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.0[117784]: debug 2023-11-16T13:31:59.254+0000 7faa42548080 -1 osd.0 0 log_to_monitors {default=true} 2023-11-16T13:31:59.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:31:58.110358+0000 mgr.smithi151.fcujra (mgr.14164) 73 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:59.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:58.997976+0000 mon.smithi151 (mon.0) 243 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:59.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:58.998819+0000 mon.smithi151 (mon.0) 244 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi151", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:31:59.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:31:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:58.999608+0000 mon.smithi151 (mon.0) 245 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:31:59.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:31:58.110358+0000 mgr.smithi151.fcujra (mgr.14164) 73 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:31:59.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:58.997976+0000 mon.smithi151 (mon.0) 243 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:31:59.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:58.998819+0000 mon.smithi151 (mon.0) 244 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi151", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:31:59.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:31:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:58.999608+0000 mon.smithi151 (mon.0) 245 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:00.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:58.998557+0000 mgr.smithi151.fcujra (mgr.14164) 74 : cephadm [INF] Reconfiguring crash.smithi151 (monmap changed)... 2023-11-16T13:32:00.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:31:59.000185+0000 mgr.smithi151.fcujra (mgr.14164) 75 : cephadm [INF] Reconfiguring daemon crash.smithi151 on smithi151 2023-11-16T13:32:00.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:31:59.258122+0000 mon.smithi151 (mon.0) 246 : audit [INF] from='osd.0 [v2:172.21.15.151:6802/3541700964,v1:172.21.15.151:6803/3541700964]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-11-16T13:32:00.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:31:58.998557+0000 mgr.smithi151.fcujra (mgr.14164) 74 : cephadm [INF] Reconfiguring crash.smithi151 (monmap changed)... 2023-11-16T13:32:00.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:31:59.000185+0000 mgr.smithi151.fcujra (mgr.14164) 75 : cephadm [INF] Reconfiguring daemon crash.smithi151 on smithi151 2023-11-16T13:32:00.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:31:59.258122+0000 mon.smithi151 (mon.0) 246 : audit [INF] from='osd.0 [v2:172.21.15.151:6802/3541700964,v1:172.21.15.151:6803/3541700964]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-11-16T13:32:01.202 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.0[117784]: debug 2023-11-16T13:32:01.179+0000 7faa38aab700 -1 osd.0 0 waiting for initial osdmap 2023-11-16T13:32:01.203 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.0[117784]: debug 2023-11-16T13:32:01.182+0000 7faa33223700 -1 osd.0 8 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2023-11-16T13:32:01.203 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:00.110599+0000 mgr.smithi151.fcujra (mgr.14164) 76 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:32:01.203 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:00.173416+0000 mon.smithi151 (mon.0) 247 : audit [INF] from='osd.0 [v2:172.21.15.151:6802/3541700964,v1:172.21.15.151:6803/3541700964]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-11-16T13:32:01.203 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:00.173465+0000 mon.smithi151 (mon.0) 248 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-11-16T13:32:01.203 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:00.173595+0000 mon.smithi151 (mon.0) 249 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:32:01.203 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:00.174059+0000 mon.smithi151 (mon.0) 250 : audit [INF] from='osd.0 [v2:172.21.15.151:6802/3541700964,v1:172.21.15.151:6803/3541700964]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:32:01.203 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:00.327979+0000 mon.smithi151 (mon.0) 251 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:01.203 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:00.328507+0000 mgr.smithi151.fcujra (mgr.14164) 77 : cephadm [INF] Reconfiguring grafana.smithi151 (dependencies changed)... 2023-11-16T13:32:01.203 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:00.332787+0000 mgr.smithi151.fcujra (mgr.14164) 78 : cephadm [INF] Reconfiguring daemon grafana.smithi151 on smithi151 2023-11-16T13:32:01.204 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:00.648708+0000 mgr.smithi151.fcujra (mgr.14164) 79 : audit [DBG] from='client.14260 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:32:01.204 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:00.649512+0000 mon.smithi151 (mon.0) 252 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:32:01.204 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:00.650815+0000 mon.smithi151 (mon.0) 253 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:32:01.204 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:00.651302+0000 mon.smithi151 (mon.0) 254 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:01.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:00.110599+0000 mgr.smithi151.fcujra (mgr.14164) 76 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:32:01.630 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:00.173416+0000 mon.smithi151 (mon.0) 247 : audit [INF] from='osd.0 [v2:172.21.15.151:6802/3541700964,v1:172.21.15.151:6803/3541700964]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-11-16T13:32:01.631 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:00.173465+0000 mon.smithi151 (mon.0) 248 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-11-16T13:32:01.631 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:00.173595+0000 mon.smithi151 (mon.0) 249 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:32:01.631 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:00.174059+0000 mon.smithi151 (mon.0) 250 : audit [INF] from='osd.0 [v2:172.21.15.151:6802/3541700964,v1:172.21.15.151:6803/3541700964]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:32:01.631 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:00.327979+0000 mon.smithi151 (mon.0) 251 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:01.631 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:00.328507+0000 mgr.smithi151.fcujra (mgr.14164) 77 : cephadm [INF] Reconfiguring grafana.smithi151 (dependencies changed)... 2023-11-16T13:32:01.631 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:00.332787+0000 mgr.smithi151.fcujra (mgr.14164) 78 : cephadm [INF] Reconfiguring daemon grafana.smithi151 on smithi151 2023-11-16T13:32:01.631 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:00.648708+0000 mgr.smithi151.fcujra (mgr.14164) 79 : audit [DBG] from='client.14260 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:32:01.631 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:00.649512+0000 mon.smithi151 (mon.0) 252 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:32:01.631 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:00.650815+0000 mon.smithi151 (mon.0) 253 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:32:01.631 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:00.651302+0000 mon.smithi151 (mon.0) 254 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:02.185 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:01.176257+0000 mon.smithi151 (mon.0) 255 : audit [INF] from='osd.0 [v2:172.21.15.151:6802/3541700964,v1:172.21.15.151:6803/3541700964]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-11-16T13:32:02.185 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:01.176314+0000 mon.smithi151 (mon.0) 256 : cluster [DBG] osdmap e8: 1 total, 0 up, 1 in 2023-11-16T13:32:02.185 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:01.177396+0000 mon.smithi151 (mon.0) 257 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:32:02.185 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:01.181420+0000 mon.smithi151 (mon.0) 258 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:32:02.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:01.176257+0000 mon.smithi151 (mon.0) 255 : audit [INF] from='osd.0 [v2:172.21.15.151:6802/3541700964,v1:172.21.15.151:6803/3541700964]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-11-16T13:32:02.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:01.176314+0000 mon.smithi151 (mon.0) 256 : cluster [DBG] osdmap e8: 1 total, 0 up, 1 in 2023-11-16T13:32:02.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:01.177396+0000 mon.smithi151 (mon.0) 257 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:32:02.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:01.181420+0000 mon.smithi151 (mon.0) 258 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:32:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:00.207328+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:32:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:00.207402+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:32:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:02.110964+0000 mgr.smithi151.fcujra (mgr.14164) 80 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:32:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:02.182910+0000 mon.smithi151 (mon.0) 259 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:32:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:02.186087+0000 mon.smithi151 (mon.0) 260 : cluster [INF] osd.0 [v2:172.21.15.151:6802/3541700964,v1:172.21.15.151:6803/3541700964] boot 2023-11-16T13:32:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:02.186117+0000 mon.smithi151 (mon.0) 261 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-11-16T13:32:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:02.186387+0000 mon.smithi151 (mon.0) 262 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:32:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:02.766349+0000 mon.smithi151 (mon.0) 263 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:02.767171+0000 mgr.smithi151.fcujra (mgr.14164) 81 : cephadm [INF] Reconfiguring prometheus.smithi151 (dependencies changed)... 2023-11-16T13:32:03.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:02.769505+0000 mgr.smithi151.fcujra (mgr.14164) 82 : cephadm [INF] Reconfiguring daemon prometheus.smithi151 on smithi151 2023-11-16T13:32:03.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:00.207328+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:32:03.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:00.207402+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:32:03.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:02.110964+0000 mgr.smithi151.fcujra (mgr.14164) 80 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-16T13:32:03.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:02.182910+0000 mon.smithi151 (mon.0) 259 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:32:03.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:02.186087+0000 mon.smithi151 (mon.0) 260 : cluster [INF] osd.0 [v2:172.21.15.151:6802/3541700964,v1:172.21.15.151:6803/3541700964] boot 2023-11-16T13:32:03.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:02.186117+0000 mon.smithi151 (mon.0) 261 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-11-16T13:32:03.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:02.186387+0000 mon.smithi151 (mon.0) 262 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:32:03.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:02.766349+0000 mon.smithi151 (mon.0) 263 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:03.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:02.767171+0000 mgr.smithi151.fcujra (mgr.14164) 81 : cephadm [INF] Reconfiguring prometheus.smithi151 (dependencies changed)... 2023-11-16T13:32:03.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:02.769505+0000 mgr.smithi151.fcujra (mgr.14164) 82 : cephadm [INF] Reconfiguring daemon prometheus.smithi151 on smithi151 2023-11-16T13:32:04.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:03.196015+0000 mon.smithi151 (mon.0) 264 : cluster [DBG] osdmap e10: 1 total, 1 up, 1 in 2023-11-16T13:32:04.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:03.592122+0000 mon.smithi151 (mon.0) 265 : audit [INF] from='client.? 172.21.15.151:0/2064340622' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9923417-ee98-4100-9df6-035b2a72db39"}]: dispatch 2023-11-16T13:32:04.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:03.598165+0000 mon.smithi151 (mon.0) 266 : audit [INF] from='client.? 172.21.15.151:0/2064340622' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a9923417-ee98-4100-9df6-035b2a72db39"}]': finished 2023-11-16T13:32:04.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:03.598205+0000 mon.smithi151 (mon.0) 267 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-11-16T13:32:04.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:03.598308+0000 mon.smithi151 (mon.0) 268 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:32:04.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:04.006985+0000 mon.smithi151 (mon.0) 269 : audit [DBG] from='client.? 172.21.15.151:0/1539918575' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:32:04.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:03.196015+0000 mon.smithi151 (mon.0) 264 : cluster [DBG] osdmap e10: 1 total, 1 up, 1 in 2023-11-16T13:32:04.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:03.592122+0000 mon.smithi151 (mon.0) 265 : audit [INF] from='client.? 172.21.15.151:0/2064340622' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9923417-ee98-4100-9df6-035b2a72db39"}]: dispatch 2023-11-16T13:32:04.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:03.598165+0000 mon.smithi151 (mon.0) 266 : audit [INF] from='client.? 172.21.15.151:0/2064340622' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a9923417-ee98-4100-9df6-035b2a72db39"}]': finished 2023-11-16T13:32:04.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:03.598205+0000 mon.smithi151 (mon.0) 267 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-11-16T13:32:04.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:03.598308+0000 mon.smithi151 (mon.0) 268 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:32:04.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:04.006985+0000 mon.smithi151 (mon.0) 269 : audit [DBG] from='client.? 172.21.15.151:0/1539918575' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:32:05.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:04.111276+0000 mgr.smithi151.fcujra (mgr.14164) 83 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:06.124 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:04.111276+0000 mgr.smithi151.fcujra (mgr.14164) 83 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:07.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:06.111635+0000 mgr.smithi151.fcujra (mgr.14164) 84 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:15.196 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:08.111919+0000 mgr.smithi151.fcujra (mgr.14164) 85 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:15.196 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:08.923226+0000 mon.smithi151 (mon.0) 270 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:08.924075+0000 mon.smithi151 (mon.0) 271 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi167", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:32:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:08.924865+0000 mon.smithi151 (mon.0) 272 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:09.773289+0000 mon.smithi151 (mon.0) 273 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:09.774118+0000 mon.smithi151 (mon.0) 274 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi167.xxnzsz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:32:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:09.774801+0000 mon.smithi151 (mon.0) 275 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:32:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:09.775334+0000 mon.smithi151 (mon.0) 276 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:09.782237+0000 mon.smithi151 (mon.0) 277 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-11-16T13:32:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:09.782820+0000 mon.smithi151 (mon.0) 278 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:08.923802+0000 mgr.smithi151.fcujra (mgr.14164) 86 : cephadm [INF] Reconfiguring crash.smithi167 (monmap changed)... 2023-11-16T13:32:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:08.925336+0000 mgr.smithi151.fcujra (mgr.14164) 87 : cephadm [INF] Reconfiguring daemon crash.smithi167 on smithi167 2023-11-16T13:32:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:09.773841+0000 mgr.smithi151.fcujra (mgr.14164) 88 : cephadm [INF] Reconfiguring mgr.smithi167.xxnzsz (monmap changed)... 2023-11-16T13:32:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:09.775770+0000 mgr.smithi151.fcujra (mgr.14164) 89 : cephadm [INF] Reconfiguring daemon mgr.smithi167.xxnzsz on smithi167 2023-11-16T13:32:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:09.783267+0000 mgr.smithi151.fcujra (mgr.14164) 90 : cephadm [INF] Deploying daemon osd.1 on smithi151 2023-11-16T13:32:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:10.595092+0000 mon.smithi151 (mon.0) 279 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:10.595713+0000 mon.smithi151 (mon.0) 280 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:32:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:10.596222+0000 mon.smithi151 (mon.0) 281 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-11-16T13:32:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:10.596690+0000 mon.smithi151 (mon.0) 282 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:10.112189+0000 mgr.smithi151.fcujra (mgr.14164) 91 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:10.595518+0000 mgr.smithi151.fcujra (mgr.14164) 92 : cephadm [INF] Reconfiguring mon.smithi167 (monmap changed)... 2023-11-16T13:32:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:10.597150+0000 mgr.smithi151.fcujra (mgr.14164) 93 : cephadm [INF] Reconfiguring daemon mon.smithi167 on smithi167 2023-11-16T13:32:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:11.385685+0000 mon.smithi151 (mon.0) 283 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:11.388018+0000 mon.smithi151 (mon.0) 284 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-11-16T13:32:15.199 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:06.111635+0000 mgr.smithi151.fcujra (mgr.14164) 84 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:15.199 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:08.111919+0000 mgr.smithi151.fcujra (mgr.14164) 85 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:15.200 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:08.923226+0000 mon.smithi151 (mon.0) 270 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.200 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:08.924075+0000 mon.smithi151 (mon.0) 271 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi167", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:32:15.200 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:08.924865+0000 mon.smithi151 (mon.0) 272 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:15.200 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:09.773289+0000 mon.smithi151 (mon.0) 273 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.200 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:09.774118+0000 mon.smithi151 (mon.0) 274 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi167.xxnzsz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:32:15.200 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:09.774801+0000 mon.smithi151 (mon.0) 275 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:32:15.200 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:09.775334+0000 mon.smithi151 (mon.0) 276 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:15.200 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:09.782237+0000 mon.smithi151 (mon.0) 277 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-11-16T13:32:15.200 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:09.782820+0000 mon.smithi151 (mon.0) 278 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:15.200 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:08.923802+0000 mgr.smithi151.fcujra (mgr.14164) 86 : cephadm [INF] Reconfiguring crash.smithi167 (monmap changed)... 2023-11-16T13:32:15.200 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:08.925336+0000 mgr.smithi151.fcujra (mgr.14164) 87 : cephadm [INF] Reconfiguring daemon crash.smithi167 on smithi167 2023-11-16T13:32:15.200 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:09.773841+0000 mgr.smithi151.fcujra (mgr.14164) 88 : cephadm [INF] Reconfiguring mgr.smithi167.xxnzsz (monmap changed)... 2023-11-16T13:32:15.201 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:09.775770+0000 mgr.smithi151.fcujra (mgr.14164) 89 : cephadm [INF] Reconfiguring daemon mgr.smithi167.xxnzsz on smithi167 2023-11-16T13:32:15.201 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:09.783267+0000 mgr.smithi151.fcujra (mgr.14164) 90 : cephadm [INF] Deploying daemon osd.1 on smithi151 2023-11-16T13:32:15.201 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:10.595092+0000 mon.smithi151 (mon.0) 279 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.201 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:10.595713+0000 mon.smithi151 (mon.0) 280 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:32:15.201 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:10.596222+0000 mon.smithi151 (mon.0) 281 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-11-16T13:32:15.201 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:10.596690+0000 mon.smithi151 (mon.0) 282 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:15.256 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:10.112189+0000 mgr.smithi151.fcujra (mgr.14164) 91 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:15.256 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:10.595518+0000 mgr.smithi151.fcujra (mgr.14164) 92 : cephadm [INF] Reconfiguring mon.smithi167 (monmap changed)... 2023-11-16T13:32:15.256 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:10.597150+0000 mgr.smithi151.fcujra (mgr.14164) 93 : cephadm [INF] Reconfiguring daemon mon.smithi167 on smithi167 2023-11-16T13:32:15.256 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:11.385685+0000 mon.smithi151 (mon.0) 283 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.256 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:11.388018+0000 mon.smithi151 (mon.0) 284 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-11-16T13:32:15.256 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:11.388324+0000 mgr.smithi151.fcujra (mgr.14164) 94 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-11-16T13:32:15.256 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.250479+0000 mon.smithi151 (mon.0) 285 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.257 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.256206+0000 mon.smithi151 (mon.0) 286 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.257 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.269160+0000 mon.smithi151 (mon.0) 287 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.151:9093"}]: dispatch 2023-11-16T13:32:15.257 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.275384+0000 mon.smithi151 (mon.0) 288 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.257 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.278172+0000 mon.smithi151 (mon.0) 289 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-11-16T13:32:15.257 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.284566+0000 mon.smithi151 (mon.0) 290 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.151:3000"}]: dispatch 2023-11-16T13:32:15.257 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.290613+0000 mon.smithi151 (mon.0) 291 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.257 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.293092+0000 mon.smithi151 (mon.0) 292 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-11-16T13:32:15.257 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.297230+0000 mon.smithi151 (mon.0) 293 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.151:9095"}]: dispatch 2023-11-16T13:32:15.257 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.302899+0000 mon.smithi151 (mon.0) 294 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.257 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.304263+0000 mon.smithi151 (mon.0) 295 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:15.257 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.305070+0000 mon.smithi151 (mon.0) 296 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:15.257 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.305810+0000 mon.smithi151 (mon.0) 297 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:15.257 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:12.112724+0000 mgr.smithi151.fcujra (mgr.14164) 95 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:15.258 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.269570+0000 mgr.smithi151.fcujra (mgr.14164) 96 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.151:9093"}]: dispatch 2023-11-16T13:32:15.258 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.278593+0000 mgr.smithi151.fcujra (mgr.14164) 97 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-11-16T13:32:15.258 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.284880+0000 mgr.smithi151.fcujra (mgr.14164) 98 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.151:3000"}]: dispatch 2023-11-16T13:32:15.258 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.293557+0000 mgr.smithi151.fcujra (mgr.14164) 99 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-11-16T13:32:15.258 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:12.297506+0000 mgr.smithi151.fcujra (mgr.14164) 100 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.151:9095"}]: dispatch 2023-11-16T13:32:15.258 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:13.360740+0000 mon.smithi151 (mon.0) 298 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.259 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 1 on host 'smithi151' 2023-11-16T13:32:15.259 DEBUG:teuthology.orchestra.run.smithi151:osd.1> sudo journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.1.service 2023-11-16T13:32:15.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:11.388324+0000 mgr.smithi151.fcujra (mgr.14164) 94 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-11-16T13:32:15.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.250479+0000 mon.smithi151 (mon.0) 285 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.256206+0000 mon.smithi151 (mon.0) 286 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.269160+0000 mon.smithi151 (mon.0) 287 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.151:9093"}]: dispatch 2023-11-16T13:32:15.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.275384+0000 mon.smithi151 (mon.0) 288 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.278172+0000 mon.smithi151 (mon.0) 289 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-11-16T13:32:15.260 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.284566+0000 mon.smithi151 (mon.0) 290 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.151:3000"}]: dispatch 2023-11-16T13:32:15.261 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.290613+0000 mon.smithi151 (mon.0) 291 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.286 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.293092+0000 mon.smithi151 (mon.0) 292 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-11-16T13:32:15.286 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.297230+0000 mon.smithi151 (mon.0) 293 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.151:9095"}]: dispatch 2023-11-16T13:32:15.286 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.302899+0000 mon.smithi151 (mon.0) 294 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.287 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.304263+0000 mon.smithi151 (mon.0) 295 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:15.287 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.305070+0000 mon.smithi151 (mon.0) 296 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:15.287 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.305810+0000 mon.smithi151 (mon.0) 297 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:15.287 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:12.112724+0000 mgr.smithi151.fcujra (mgr.14164) 95 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:15.287 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.269570+0000 mgr.smithi151.fcujra (mgr.14164) 96 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.151:9093"}]: dispatch 2023-11-16T13:32:15.287 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.278593+0000 mgr.smithi151.fcujra (mgr.14164) 97 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-11-16T13:32:15.287 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.284880+0000 mgr.smithi151.fcujra (mgr.14164) 98 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.151:3000"}]: dispatch 2023-11-16T13:32:15.287 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.293557+0000 mgr.smithi151.fcujra (mgr.14164) 99 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-11-16T13:32:15.287 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:12.297506+0000 mgr.smithi151.fcujra (mgr.14164) 100 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.151:9095"}]: dispatch 2023-11-16T13:32:15.287 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:13.360740+0000 mon.smithi151 (mon.0) 298 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.290 INFO:tasks.cephadm:Deploying osd.2 on smithi151 with /dev/vg_nvme/lv_2... 2023-11-16T13:32:15.290 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-11-16T13:32:15.320 INFO:journalctl@ceph.osd.1.smithi151.stdout:-- Logs begin at Thu 2023-11-16 13:20:07 UTC. -- 2023-11-16T13:32:15.483 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:32:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.1[121057]: debug 2023-11-16T13:32:15.418+0000 7ff227141080 -1 osd.1 0 log_to_monitors {default=true} 2023-11-16T13:32:15.791 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:14.113016+0000 mgr.smithi151.fcujra (mgr.14164) 101 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:15.791 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:14.597978+0000 mon.smithi151 (mon.0) 299 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:15.792 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:15.422643+0000 mon.smithi151 (mon.0) 300 : audit [INF] from='osd.1 [v2:172.21.15.151:6810/3233915009,v1:172.21.15.151:6811/3233915009]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-11-16T13:32:16.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:14.113016+0000 mgr.smithi151.fcujra (mgr.14164) 101 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:16.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:14.597978+0000 mon.smithi151 (mon.0) 299 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:16.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:15.422643+0000 mon.smithi151 (mon.0) 300 : audit [INF] from='osd.1 [v2:172.21.15.151:6810/3233915009,v1:172.21.15.151:6811/3233915009]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-11-16T13:32:16.859 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-11-16T13:32:16.859 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-11-16T13:32:16.859 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2023-11-16T13:32:16.859 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2023-11-16T13:32:16.859 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111326 s, 942 MB/s 2023-11-16T13:32:16.860 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2023-11-16T13:32:16.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:15.893205+0000 mon.smithi151 (mon.0) 301 : audit [INF] from='osd.1 [v2:172.21.15.151:6810/3233915009,v1:172.21.15.151:6811/3233915009]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-11-16T13:32:16.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:15.893253+0000 mon.smithi151 (mon.0) 302 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-11-16T13:32:16.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:15.893343+0000 mon.smithi151 (mon.0) 303 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:32:16.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:15.893673+0000 mon.smithi151 (mon.0) 304 : audit [INF] from='osd.1 [v2:172.21.15.151:6810/3233915009,v1:172.21.15.151:6811/3233915009]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:32:16.984 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:32:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.1[121057]: debug 2023-11-16T13:32:16.897+0000 7ff21d6a4700 -1 osd.1 0 waiting for initial osdmap 2023-11-16T13:32:16.984 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:32:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.1[121057]: debug 2023-11-16T13:32:16.900+0000 7ff219e20700 -1 osd.1 13 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2023-11-16T13:32:17.233 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch daemon add osd smithi151:vg_nvme/lv_2 2023-11-16T13:32:17.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:15.893205+0000 mon.smithi151 (mon.0) 301 : audit [INF] from='osd.1 [v2:172.21.15.151:6810/3233915009,v1:172.21.15.151:6811/3233915009]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-11-16T13:32:17.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:15.893253+0000 mon.smithi151 (mon.0) 302 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-11-16T13:32:17.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:15.893343+0000 mon.smithi151 (mon.0) 303 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:32:17.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:15.893673+0000 mon.smithi151 (mon.0) 304 : audit [INF] from='osd.1 [v2:172.21.15.151:6810/3233915009,v1:172.21.15.151:6811/3233915009]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:32:18.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:16.113254+0000 mgr.smithi151.fcujra (mgr.14164) 102 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:18.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:16.894170+0000 mon.smithi151 (mon.0) 305 : audit [INF] from='osd.1 [v2:172.21.15.151:6810/3233915009,v1:172.21.15.151:6811/3233915009]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-11-16T13:32:18.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:16.894219+0000 mon.smithi151 (mon.0) 306 : cluster [DBG] osdmap e13: 2 total, 1 up, 2 in 2023-11-16T13:32:18.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:16.894579+0000 mon.smithi151 (mon.0) 307 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:32:18.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:16.898296+0000 mon.smithi151 (mon.0) 308 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:32:18.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:17.046221+0000 mon.smithi151 (mon.0) 309 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:18.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:17.047048+0000 mon.smithi151 (mon.0) 310 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:18.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:17.047667+0000 mon.smithi151 (mon.0) 311 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:18.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:17.051455+0000 mon.smithi151 (mon.0) 312 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:18.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:17.055145+0000 mon.smithi151 (mon.0) 313 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:18.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:17.057906+0000 mon.smithi151 (mon.0) 314 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:18.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:17.058693+0000 mon.smithi151 (mon.0) 315 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:18.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:17.059278+0000 mon.smithi151 (mon.0) 316 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:18.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:16.113254+0000 mgr.smithi151.fcujra (mgr.14164) 102 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:18.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:16.894170+0000 mon.smithi151 (mon.0) 305 : audit [INF] from='osd.1 [v2:172.21.15.151:6810/3233915009,v1:172.21.15.151:6811/3233915009]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-11-16T13:32:18.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:16.894219+0000 mon.smithi151 (mon.0) 306 : cluster [DBG] osdmap e13: 2 total, 1 up, 2 in 2023-11-16T13:32:18.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:16.894579+0000 mon.smithi151 (mon.0) 307 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:32:18.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:16.898296+0000 mon.smithi151 (mon.0) 308 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:32:18.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:17.046221+0000 mon.smithi151 (mon.0) 309 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:18.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:17.047048+0000 mon.smithi151 (mon.0) 310 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:18.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:17.047667+0000 mon.smithi151 (mon.0) 311 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:18.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:17.051455+0000 mon.smithi151 (mon.0) 312 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:18.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:17.055145+0000 mon.smithi151 (mon.0) 313 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:18.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:17.057906+0000 mon.smithi151 (mon.0) 314 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:18.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:17.058693+0000 mon.smithi151 (mon.0) 315 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:18.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:17.059278+0000 mon.smithi151 (mon.0) 316 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:19.214 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:16.452452+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:32:19.214 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:16.452547+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:32:19.214 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:17.041372+0000 mgr.smithi151.fcujra (mgr.14164) 103 : cephadm [INF] Detected new or changed devices on smithi151 2023-11-16T13:32:19.214 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:17.048068+0000 mgr.smithi151.fcujra (mgr.14164) 104 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 6449M 2023-11-16T13:32:19.214 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:17.899296+0000 mon.smithi151 (mon.0) 317 : cluster [INF] osd.1 [v2:172.21.15.151:6810/3233915009,v1:172.21.15.151:6811/3233915009] boot 2023-11-16T13:32:19.215 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:17.899330+0000 mon.smithi151 (mon.0) 318 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-11-16T13:32:19.215 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:17.899838+0000 mon.smithi151 (mon.0) 319 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:32:19.215 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:18.128345+0000 mon.smithi151 (mon.0) 320 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:19.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:16.452452+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:32:19.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:16.452547+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:32:19.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:17.041372+0000 mgr.smithi151.fcujra (mgr.14164) 103 : cephadm [INF] Detected new or changed devices on smithi151 2023-11-16T13:32:19.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:17.048068+0000 mgr.smithi151.fcujra (mgr.14164) 104 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 6449M 2023-11-16T13:32:19.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:17.899296+0000 mon.smithi151 (mon.0) 317 : cluster [INF] osd.1 [v2:172.21.15.151:6810/3233915009,v1:172.21.15.151:6811/3233915009] boot 2023-11-16T13:32:19.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:17.899330+0000 mon.smithi151 (mon.0) 318 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-11-16T13:32:19.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:17.899838+0000 mon.smithi151 (mon.0) 319 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:32:19.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:18.128345+0000 mon.smithi151 (mon.0) 320 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:20.099 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:18.113608+0000 mgr.smithi151.fcujra (mgr.14164) 105 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:20.099 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:18.956880+0000 mgr.smithi151.fcujra (mgr.14164) 106 : audit [DBG] from='client.14278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:32:20.099 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:18.957611+0000 mon.smithi151 (mon.0) 321 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:32:20.099 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:18.958715+0000 mon.smithi151 (mon.0) 322 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:32:20.099 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:18.959203+0000 mon.smithi151 (mon.0) 323 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:20.099 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:19.129234+0000 mon.smithi151 (mon.0) 324 : cluster [DBG] osdmap e15: 2 total, 2 up, 2 in 2023-11-16T13:32:20.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:18.113608+0000 mgr.smithi151.fcujra (mgr.14164) 105 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-11-16T13:32:20.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:18.956880+0000 mgr.smithi151.fcujra (mgr.14164) 106 : audit [DBG] from='client.14278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:32:20.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:18.957611+0000 mon.smithi151 (mon.0) 321 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:32:20.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:18.958715+0000 mon.smithi151 (mon.0) 322 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:32:20.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:18.959203+0000 mon.smithi151 (mon.0) 323 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:20.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:19.129234+0000 mon.smithi151 (mon.0) 324 : cluster [DBG] osdmap e15: 2 total, 2 up, 2 in 2023-11-16T13:32:21.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:20.113867+0000 mgr.smithi151.fcujra (mgr.14164) 107 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:21.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:20.234893+0000 mon.smithi151 (mon.0) 325 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:21.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:20.795559+0000 mon.smithi151 (mon.0) 326 : audit [INF] from='client.? 172.21.15.151:0/1746389872' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b581b07-525f-4aca-97ce-089a9cec4901"}]: dispatch 2023-11-16T13:32:21.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:20.799713+0000 mon.smithi151 (mon.0) 327 : audit [INF] from='client.? 172.21.15.151:0/1746389872' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5b581b07-525f-4aca-97ce-089a9cec4901"}]': finished 2023-11-16T13:32:21.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:20.799749+0000 mon.smithi151 (mon.0) 328 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-11-16T13:32:21.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:20.799835+0000 mon.smithi151 (mon.0) 329 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:32:21.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:21.219409+0000 mon.smithi151 (mon.0) 330 : audit [DBG] from='client.? 172.21.15.151:0/1031554783' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:32:21.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:20.113867+0000 mgr.smithi151.fcujra (mgr.14164) 107 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:21.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:20.234893+0000 mon.smithi151 (mon.0) 325 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:21.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:20.795559+0000 mon.smithi151 (mon.0) 326 : audit [INF] from='client.? 172.21.15.151:0/1746389872' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b581b07-525f-4aca-97ce-089a9cec4901"}]: dispatch 2023-11-16T13:32:21.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:20.799713+0000 mon.smithi151 (mon.0) 327 : audit [INF] from='client.? 172.21.15.151:0/1746389872' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5b581b07-525f-4aca-97ce-089a9cec4901"}]': finished 2023-11-16T13:32:21.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:20.799749+0000 mon.smithi151 (mon.0) 328 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-11-16T13:32:21.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:20.799835+0000 mon.smithi151 (mon.0) 329 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:32:21.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:21.219409+0000 mon.smithi151 (mon.0) 330 : audit [DBG] from='client.? 172.21.15.151:0/1031554783' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:32:23.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:22.114072+0000 mgr.smithi151.fcujra (mgr.14164) 108 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:23.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:22.240316+0000 mon.smithi151 (mon.0) 331 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:32:23.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:22.241701+0000 mon.smithi151 (mon.0) 332 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:32:23.616 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:22.114072+0000 mgr.smithi151.fcujra (mgr.14164) 108 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:23.616 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:22.240316+0000 mon.smithi151 (mon.0) 331 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:32:23.616 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:22.241701+0000 mon.smithi151 (mon.0) 332 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:32:25.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:25 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:24.114309+0000 mgr.smithi151.fcujra (mgr.14164) 109 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:25.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:25 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:24.114309+0000 mgr.smithi151.fcujra (mgr.14164) 109 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:26.698 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:25.365408+0000 mgr.smithi151.fcujra (mgr.14164) 110 : cephadm [INF] Detected new or changed devices on smithi151 2023-11-16T13:32:26.699 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:25.370470+0000 mon.smithi151 (mon.0) 333 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:26.699 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:25.375815+0000 mon.smithi151 (mon.0) 334 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:26.699 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:26.084891+0000 mon.smithi151 (mon.0) 335 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-11-16T13:32:26.699 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:26.085423+0000 mon.smithi151 (mon.0) 336 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:26.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:25.365408+0000 mgr.smithi151.fcujra (mgr.14164) 110 : cephadm [INF] Detected new or changed devices on smithi151 2023-11-16T13:32:26.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:25.370470+0000 mon.smithi151 (mon.0) 333 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:26.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:25.375815+0000 mon.smithi151 (mon.0) 334 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:26.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:26.084891+0000 mon.smithi151 (mon.0) 335 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-11-16T13:32:26.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:26.085423+0000 mon.smithi151 (mon.0) 336 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:27.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:26.085832+0000 mgr.smithi151.fcujra (mgr.14164) 111 : cephadm [INF] Deploying daemon osd.2 on smithi151 2023-11-16T13:32:27.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:26.114593+0000 mgr.smithi151.fcujra (mgr.14164) 112 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:28.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:26.085832+0000 mgr.smithi151.fcujra (mgr.14164) 111 : cephadm [INF] Deploying daemon osd.2 on smithi151 2023-11-16T13:32:28.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:26.114593+0000 mgr.smithi151.fcujra (mgr.14164) 112 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:28.375 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 2 on host 'smithi151' 2023-11-16T13:32:28.752 DEBUG:teuthology.orchestra.run.smithi151:osd.2> sudo journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.2.service 2023-11-16T13:32:28.754 INFO:tasks.cephadm:Deploying osd.3 on smithi151 with /dev/vg_nvme/lv_1... 2023-11-16T13:32:28.754 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-11-16T13:32:28.788 INFO:journalctl@ceph.osd.2.smithi151.stdout:-- Logs begin at Thu 2023-11-16 13:20:07 UTC. -- 2023-11-16T13:32:28.979 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:32:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.2[124902]: debug 2023-11-16T13:32:28.832+0000 7f152a0e6080 -1 Falling back to public interface 2023-11-16T13:32:28.979 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:28.361978+0000 mon.smithi151 (mon.0) 337 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:28.979 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:28.363550+0000 mon.smithi151 (mon.0) 338 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:28.979 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:28.365603+0000 mon.smithi151 (mon.0) 339 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:28.979 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:28.365818+0000 mon.smithi151 (mon.0) 340 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:28.979 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:28.369970+0000 mon.smithi151 (mon.0) 341 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:29.266 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:28.361978+0000 mon.smithi151 (mon.0) 337 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:29.266 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:28.363550+0000 mon.smithi151 (mon.0) 338 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:29.266 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:28.365603+0000 mon.smithi151 (mon.0) 339 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:29.267 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:28.365818+0000 mon.smithi151 (mon.0) 340 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:29.267 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:28.369970+0000 mon.smithi151 (mon.0) 341 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:29.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:28.115022+0000 mgr.smithi151.fcujra (mgr.14164) 113 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:29.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:29.492299+0000 mon.smithi151 (mon.0) 342 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:30.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:28.115022+0000 mgr.smithi151.fcujra (mgr.14164) 113 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:30.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:29.492299+0000 mon.smithi151 (mon.0) 342 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:30.503 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-11-16T13:32:30.504 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-11-16T13:32:30.504 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2023-11-16T13:32:30.504 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2023-11-16T13:32:30.504 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.01311 s, 800 MB/s 2023-11-16T13:32:30.504 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2023-11-16T13:32:30.970 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch daemon add osd smithi151:vg_nvme/lv_1 2023-11-16T13:32:31.887 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:32:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.2[124902]: debug 2023-11-16T13:32:31.564+0000 7f152a0e6080 -1 osd.2 0 log_to_monitors {default=true} 2023-11-16T13:32:32.139 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:30.115274+0000 mgr.smithi151.fcujra (mgr.14164) 114 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:32.139 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:31.569426+0000 mon.smithi151 (mon.0) 343 : audit [INF] from='osd.2 [v2:172.21.15.151:6818/1911452143,v1:172.21.15.151:6819/1911452143]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-11-16T13:32:32.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:30.115274+0000 mgr.smithi151.fcujra (mgr.14164) 114 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:32.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:31.569426+0000 mon.smithi151 (mon.0) 343 : audit [INF] from='osd.2 [v2:172.21.15.151:6818/1911452143,v1:172.21.15.151:6819/1911452143]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-11-16T13:32:32.932 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:32:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.2[124902]: debug 2023-11-16T13:32:32.894+0000 7f1520649700 -1 osd.2 0 waiting for initial osdmap 2023-11-16T13:32:32.932 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:32:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.2[124902]: debug 2023-11-16T13:32:32.897+0000 7f1518dbd700 -1 osd.2 18 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2023-11-16T13:32:32.933 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:31.884574+0000 mon.smithi151 (mon.0) 344 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:32.933 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:31.891585+0000 mon.smithi151 (mon.0) 345 : audit [INF] from='osd.2 [v2:172.21.15.151:6818/1911452143,v1:172.21.15.151:6819/1911452143]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-11-16T13:32:32.933 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:31.891628+0000 mon.smithi151 (mon.0) 346 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-11-16T13:32:32.933 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:31.891741+0000 mon.smithi151 (mon.0) 347 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:32:32.933 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:31.892033+0000 mon.smithi151 (mon.0) 348 : audit [INF] from='osd.2 [v2:172.21.15.151:6818/1911452143,v1:172.21.15.151:6819/1911452143]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:32:32.933 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:32.524886+0000 mon.smithi151 (mon.0) 349 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:32:32.933 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:32.526040+0000 mon.smithi151 (mon.0) 350 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:32:32.933 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:32.526548+0000 mon.smithi151 (mon.0) 351 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:33.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:31.884574+0000 mon.smithi151 (mon.0) 344 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:33.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:31.891585+0000 mon.smithi151 (mon.0) 345 : audit [INF] from='osd.2 [v2:172.21.15.151:6818/1911452143,v1:172.21.15.151:6819/1911452143]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-11-16T13:32:33.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:31.891628+0000 mon.smithi151 (mon.0) 346 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-11-16T13:32:33.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:31.891741+0000 mon.smithi151 (mon.0) 347 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:32:33.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:31.892033+0000 mon.smithi151 (mon.0) 348 : audit [INF] from='osd.2 [v2:172.21.15.151:6818/1911452143,v1:172.21.15.151:6819/1911452143]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:32:33.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:32.524886+0000 mon.smithi151 (mon.0) 349 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:32:33.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:32.526040+0000 mon.smithi151 (mon.0) 350 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:32:33.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:32.526548+0000 mon.smithi151 (mon.0) 351 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:34.223 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:32.115536+0000 mgr.smithi151.fcujra (mgr.14164) 115 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:34.223 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:32.524108+0000 mgr.smithi151.fcujra (mgr.14164) 116 : audit [DBG] from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:32:34.223 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:32.892322+0000 mon.smithi151 (mon.0) 352 : audit [INF] from='osd.2 [v2:172.21.15.151:6818/1911452143,v1:172.21.15.151:6819/1911452143]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-11-16T13:32:34.223 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:32.892368+0000 mon.smithi151 (mon.0) 353 : cluster [DBG] osdmap e18: 3 total, 2 up, 3 in 2023-11-16T13:32:34.223 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:32.892626+0000 mon.smithi151 (mon.0) 354 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:32:34.223 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:32.895604+0000 mon.smithi151 (mon.0) 355 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:32:34.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:32.115536+0000 mgr.smithi151.fcujra (mgr.14164) 115 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:34.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:32.524108+0000 mgr.smithi151.fcujra (mgr.14164) 116 : audit [DBG] from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:32:34.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:32.892322+0000 mon.smithi151 (mon.0) 352 : audit [INF] from='osd.2 [v2:172.21.15.151:6818/1911452143,v1:172.21.15.151:6819/1911452143]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-11-16T13:32:34.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:32.892368+0000 mon.smithi151 (mon.0) 353 : cluster [DBG] osdmap e18: 3 total, 2 up, 3 in 2023-11-16T13:32:34.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:32.892626+0000 mon.smithi151 (mon.0) 354 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:32:34.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:32.895604+0000 mon.smithi151 (mon.0) 355 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:32:35.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:32.525636+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:32:35.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:32.525722+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:32:35.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:33.895645+0000 mon.smithi151 (mon.0) 356 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:32:35.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:33.902963+0000 mon.smithi151 (mon.0) 357 : cluster [INF] osd.2 [v2:172.21.15.151:6818/1911452143,v1:172.21.15.151:6819/1911452143] boot 2023-11-16T13:32:35.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:33.902993+0000 mon.smithi151 (mon.0) 358 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-11-16T13:32:35.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:33.903253+0000 mon.smithi151 (mon.0) 359 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:32:35.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:34.311410+0000 mon.smithi151 (mon.0) 360 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:35.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:34.312328+0000 mon.smithi151 (mon.0) 361 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:35.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:34.312922+0000 mon.smithi151 (mon.0) 362 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:35.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:34.313509+0000 mon.smithi151 (mon.0) 363 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:35.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:34.319808+0000 mon.smithi151 (mon.0) 364 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:35.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:34.325249+0000 mon.smithi151 (mon.0) 365 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:35.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:34.329762+0000 mon.smithi151 (mon.0) 366 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:35.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:34.330721+0000 mon.smithi151 (mon.0) 367 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:35.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:34.331242+0000 mon.smithi151 (mon.0) 368 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:35.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:32.525636+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:32:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:32.525722+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:32:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:33.895645+0000 mon.smithi151 (mon.0) 356 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:32:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:33.902963+0000 mon.smithi151 (mon.0) 357 : cluster [INF] osd.2 [v2:172.21.15.151:6818/1911452143,v1:172.21.15.151:6819/1911452143] boot 2023-11-16T13:32:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:33.902993+0000 mon.smithi151 (mon.0) 358 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-11-16T13:32:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:33.903253+0000 mon.smithi151 (mon.0) 359 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:32:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:34.311410+0000 mon.smithi151 (mon.0) 360 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:34.312328+0000 mon.smithi151 (mon.0) 361 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:34.312922+0000 mon.smithi151 (mon.0) 362 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:34.313509+0000 mon.smithi151 (mon.0) 363 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:34.319808+0000 mon.smithi151 (mon.0) 364 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:34.325249+0000 mon.smithi151 (mon.0) 365 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:34.329762+0000 mon.smithi151 (mon.0) 366 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:35.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:34.330721+0000 mon.smithi151 (mon.0) 367 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:35.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:34.331242+0000 mon.smithi151 (mon.0) 368 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:36.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:34.115819+0000 mgr.smithi151.fcujra (mgr.14164) 117 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:36.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:34.314086+0000 mgr.smithi151.fcujra (mgr.14164) 118 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 4299M 2023-11-16T13:32:36.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:34.980648+0000 mon.smithi151 (mon.0) 369 : audit [INF] from='client.? 172.21.15.151:0/592531372' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac93c609-8473-40fc-9dc5-bea25b31d2c0"}]: dispatch 2023-11-16T13:32:36.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:34.985131+0000 mon.smithi151 (mon.0) 370 : audit [INF] from='client.? 172.21.15.151:0/592531372' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ac93c609-8473-40fc-9dc5-bea25b31d2c0"}]': finished 2023-11-16T13:32:36.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:34.985174+0000 mon.smithi151 (mon.0) 371 : cluster [DBG] osdmap e20: 4 total, 3 up, 4 in 2023-11-16T13:32:36.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:34.985290+0000 mon.smithi151 (mon.0) 372 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:36.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:35.412283+0000 mon.smithi151 (mon.0) 373 : audit [DBG] from='client.? 172.21.15.151:0/3377679327' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:32:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:34.115819+0000 mgr.smithi151.fcujra (mgr.14164) 117 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-11-16T13:32:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:34.314086+0000 mgr.smithi151.fcujra (mgr.14164) 118 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 4299M 2023-11-16T13:32:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:34.980648+0000 mon.smithi151 (mon.0) 369 : audit [INF] from='client.? 172.21.15.151:0/592531372' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac93c609-8473-40fc-9dc5-bea25b31d2c0"}]: dispatch 2023-11-16T13:32:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:34.985131+0000 mon.smithi151 (mon.0) 370 : audit [INF] from='client.? 172.21.15.151:0/592531372' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ac93c609-8473-40fc-9dc5-bea25b31d2c0"}]': finished 2023-11-16T13:32:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:34.985174+0000 mon.smithi151 (mon.0) 371 : cluster [DBG] osdmap e20: 4 total, 3 up, 4 in 2023-11-16T13:32:36.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:34.985290+0000 mon.smithi151 (mon.0) 372 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:36.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:35.412283+0000 mon.smithi151 (mon.0) 373 : audit [DBG] from='client.? 172.21.15.151:0/3377679327' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:32:37.942 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:36.116068+0000 mgr.smithi151.fcujra (mgr.14164) 119 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-11-16T13:32:37.942 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:36.714670+0000 mon.smithi151 (mon.0) 374 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:37.942 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:36.717851+0000 mon.smithi151 (mon.0) 375 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:38.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:36.116068+0000 mgr.smithi151.fcujra (mgr.14164) 119 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-11-16T13:32:38.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:36.714670+0000 mon.smithi151 (mon.0) 374 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:38.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:36.717851+0000 mon.smithi151 (mon.0) 375 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:39.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:39 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:38.116352+0000 mgr.smithi151.fcujra (mgr.14164) 120 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-11-16T13:32:39.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:39 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:39.675976+0000 mon.smithi151 (mon.0) 376 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-11-16T13:32:39.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:39 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:39.676545+0000 mon.smithi151 (mon.0) 377 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:40.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:39 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:38.116352+0000 mgr.smithi151.fcujra (mgr.14164) 120 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-11-16T13:32:40.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:39 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:39.675976+0000 mon.smithi151 (mon.0) 376 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-11-16T13:32:40.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:39 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:39.676545+0000 mon.smithi151 (mon.0) 377 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:40.943 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:39.677007+0000 mgr.smithi151.fcujra (mgr.14164) 121 : cephadm [INF] Deploying daemon osd.3 on smithi151 2023-11-16T13:32:41.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:39.677007+0000 mgr.smithi151.fcujra (mgr.14164) 121 : cephadm [INF] Deploying daemon osd.3 on smithi151 2023-11-16T13:32:41.948 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 3 on host 'smithi151' 2023-11-16T13:32:42.130 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:41 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:40.116756+0000 mgr.smithi151.fcujra (mgr.14164) 122 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-11-16T13:32:42.320 DEBUG:teuthology.orchestra.run.smithi151:osd.3> sudo journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.3.service 2023-11-16T13:32:42.322 INFO:tasks.cephadm:Deploying osd.4 on smithi167 with /dev/vg_nvme/lv_4... 2023-11-16T13:32:42.323 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-11-16T13:32:42.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:41 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:40.116756+0000 mgr.smithi151.fcujra (mgr.14164) 122 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-11-16T13:32:42.353 INFO:journalctl@ceph.osd.3.smithi151.stdout:-- Logs begin at Thu 2023-11-16 13:20:07 UTC. -- 2023-11-16T13:32:42.733 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:32:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.3[128565]: debug 2023-11-16T13:32:42.400+0000 7f12d321f080 -1 Falling back to public interface 2023-11-16T13:32:42.933 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:41.933835+0000 mon.smithi151 (mon.0) 378 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:42.933 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:41.935548+0000 mon.smithi151 (mon.0) 379 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:42.933 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:41.937730+0000 mon.smithi151 (mon.0) 380 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:42.933 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:41.937969+0000 mon.smithi151 (mon.0) 381 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:42.933 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:41.944628+0000 mon.smithi151 (mon.0) 382 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:42.933 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:41.948180+0000 mon.smithi151 (mon.0) 383 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2023-11-16T13:32:43.218 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:41.933835+0000 mon.smithi151 (mon.0) 378 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:43.218 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:41.935548+0000 mon.smithi151 (mon.0) 379 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:43.218 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:41.937730+0000 mon.smithi151 (mon.0) 380 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:43.218 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:41.937969+0000 mon.smithi151 (mon.0) 381 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:43.218 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:41.944628+0000 mon.smithi151 (mon.0) 382 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:43.218 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:41.948180+0000 mon.smithi151 (mon.0) 383 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2023-11-16T13:32:43.686 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-11-16T13:32:43.687 INFO:teuthology.orchestra.run.smithi167.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-11-16T13:32:43.687 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10+0 records in 2023-11-16T13:32:43.687 INFO:teuthology.orchestra.run.smithi167.stderr:10+0 records out 2023-11-16T13:32:43.687 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111153 s, 943 MB/s 2023-11-16T13:32:43.687 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping successful for: 2023-11-16T13:32:43.986 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:42.116974+0000 mgr.smithi151.fcujra (mgr.14164) 123 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-11-16T13:32:43.986 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:42.947746+0000 mon.smithi151 (mon.0) 384 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]': finished 2023-11-16T13:32:43.987 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:42.947793+0000 mon.smithi151 (mon.0) 385 : cluster [DBG] osdmap e21: 4 total, 3 up, 4 in 2023-11-16T13:32:43.987 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:42.947896+0000 mon.smithi151 (mon.0) 386 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:43.987 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:42.948471+0000 mon.smithi151 (mon.0) 387 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-11-16T13:32:44.023 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch daemon add osd smithi167:vg_nvme/lv_4 2023-11-16T13:32:44.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:42.116974+0000 mgr.smithi151.fcujra (mgr.14164) 123 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-11-16T13:32:44.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:42.947746+0000 mon.smithi151 (mon.0) 384 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]': finished 2023-11-16T13:32:44.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:42.947793+0000 mon.smithi151 (mon.0) 385 : cluster [DBG] osdmap e21: 4 total, 3 up, 4 in 2023-11-16T13:32:44.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:42.947896+0000 mon.smithi151 (mon.0) 386 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:44.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:42.948471+0000 mon.smithi151 (mon.0) 387 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-11-16T13:32:45.234 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:32:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.3[128565]: debug 2023-11-16T13:32:45.113+0000 7f12d321f080 -1 osd.3 0 log_to_monitors {default=true} 2023-11-16T13:32:45.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:43.952024+0000 mon.smithi151 (mon.0) 388 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-11-16T13:32:45.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:43.952094+0000 mon.smithi151 (mon.0) 389 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-11-16T13:32:45.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:43.952196+0000 mon.smithi151 (mon.0) 390 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:45.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:44.117473+0000 mgr.smithi151.fcujra (mgr.14164) 124 : cluster [DBG] pgmap v82: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-11-16T13:32:45.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:44.657307+0000 mon.smithi151 (mon.0) 391 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:45.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:44 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:43.952024+0000 mon.smithi151 (mon.0) 388 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-11-16T13:32:45.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:44 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:43.952094+0000 mon.smithi151 (mon.0) 389 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-11-16T13:32:45.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:44 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:43.952196+0000 mon.smithi151 (mon.0) 390 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:45.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:44 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:44.117473+0000 mgr.smithi151.fcujra (mgr.14164) 124 : cluster [DBG] pgmap v82: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-11-16T13:32:45.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:44 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:44.657307+0000 mon.smithi151 (mon.0) 391 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:46.156 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:44.958858+0000 mon.smithi151 (mon.0) 392 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-11-16T13:32:46.156 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:44.958977+0000 mon.smithi151 (mon.0) 393 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:46.157 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:45.117559+0000 mon.smithi151 (mon.0) 394 : audit [INF] from='osd.3 [v2:172.21.15.151:6826/3194408863,v1:172.21.15.151:6827/3194408863]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-11-16T13:32:46.157 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:45.308362+0000 mgr.smithi151.fcujra (mgr.14164) 125 : audit [DBG] from='client.24139 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:32:46.157 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:45.309147+0000 mon.smithi151 (mon.0) 395 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:32:46.157 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:45.310542+0000 mon.smithi151 (mon.0) 396 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:32:46.157 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:45.311028+0000 mon.smithi151 (mon.0) 397 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:46.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:44.958858+0000 mon.smithi151 (mon.0) 392 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-11-16T13:32:46.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:44.958977+0000 mon.smithi151 (mon.0) 393 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:46.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:45.117559+0000 mon.smithi151 (mon.0) 394 : audit [INF] from='osd.3 [v2:172.21.15.151:6826/3194408863,v1:172.21.15.151:6827/3194408863]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-11-16T13:32:46.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:45.308362+0000 mgr.smithi151.fcujra (mgr.14164) 125 : audit [DBG] from='client.24139 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:32:46.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:45.309147+0000 mon.smithi151 (mon.0) 395 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:32:46.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:45.310542+0000 mon.smithi151 (mon.0) 396 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:32:46.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:45.311028+0000 mon.smithi151 (mon.0) 397 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:47.233 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.3[128565]: debug 2023-11-16T13:32:46.814+0000 7f12c9782700 -1 osd.3 0 waiting for initial osdmap 2023-11-16T13:32:47.234 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.3[128565]: debug 2023-11-16T13:32:46.818+0000 7f12c5efe700 -1 osd.3 25 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2023-11-16T13:32:47.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:45.964110+0000 mon.smithi151 (mon.0) 398 : audit [INF] from='osd.3 [v2:172.21.15.151:6826/3194408863,v1:172.21.15.151:6827/3194408863]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-11-16T13:32:47.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:45.964176+0000 mon.smithi151 (mon.0) 399 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-11-16T13:32:47.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:45.964275+0000 mon.smithi151 (mon.0) 400 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:47.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:45.964576+0000 mon.smithi151 (mon.0) 401 : audit [INF] from='osd.3 [v2:172.21.15.151:6826/3194408863,v1:172.21.15.151:6827/3194408863]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:32:47.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:46.117795+0000 mgr.smithi151.fcujra (mgr.14164) 126 : cluster [DBG] pgmap v85: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-11-16T13:32:47.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:46.807992+0000 mon.smithi167 (mon.1) 2 : audit [INF] from='client.? 172.21.15.167:0/611056982' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b187f517-360e-4ded-9afe-b6ca35c88906"}]: dispatch 2023-11-16T13:32:47.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:46.808476+0000 mon.smithi151 (mon.0) 402 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b187f517-360e-4ded-9afe-b6ca35c88906"}]: dispatch 2023-11-16T13:32:47.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:46.812969+0000 mon.smithi151 (mon.0) 403 : audit [INF] from='osd.3 [v2:172.21.15.151:6826/3194408863,v1:172.21.15.151:6827/3194408863]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-11-16T13:32:47.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:46.813019+0000 mon.smithi151 (mon.0) 404 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b187f517-360e-4ded-9afe-b6ca35c88906"}]': finished 2023-11-16T13:32:47.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:46.813053+0000 mon.smithi151 (mon.0) 405 : cluster [DBG] osdmap e25: 5 total, 3 up, 5 in 2023-11-16T13:32:47.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:46.813166+0000 mon.smithi151 (mon.0) 406 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:47.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:46.813473+0000 mon.smithi151 (mon.0) 407 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:47.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:46.816536+0000 mon.smithi151 (mon.0) 408 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:47.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:45.964110+0000 mon.smithi151 (mon.0) 398 : audit [INF] from='osd.3 [v2:172.21.15.151:6826/3194408863,v1:172.21.15.151:6827/3194408863]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-11-16T13:32:47.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:45.964176+0000 mon.smithi151 (mon.0) 399 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-11-16T13:32:47.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:45.964275+0000 mon.smithi151 (mon.0) 400 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:47.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:45.964576+0000 mon.smithi151 (mon.0) 401 : audit [INF] from='osd.3 [v2:172.21.15.151:6826/3194408863,v1:172.21.15.151:6827/3194408863]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:32:47.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:46.117795+0000 mgr.smithi151.fcujra (mgr.14164) 126 : cluster [DBG] pgmap v85: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-11-16T13:32:47.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:46.807992+0000 mon.smithi167 (mon.1) 2 : audit [INF] from='client.? 172.21.15.167:0/611056982' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b187f517-360e-4ded-9afe-b6ca35c88906"}]: dispatch 2023-11-16T13:32:47.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:46.808476+0000 mon.smithi151 (mon.0) 402 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b187f517-360e-4ded-9afe-b6ca35c88906"}]: dispatch 2023-11-16T13:32:47.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:46.812969+0000 mon.smithi151 (mon.0) 403 : audit [INF] from='osd.3 [v2:172.21.15.151:6826/3194408863,v1:172.21.15.151:6827/3194408863]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-11-16T13:32:47.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:46.813019+0000 mon.smithi151 (mon.0) 404 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b187f517-360e-4ded-9afe-b6ca35c88906"}]': finished 2023-11-16T13:32:47.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:46.813053+0000 mon.smithi151 (mon.0) 405 : cluster [DBG] osdmap e25: 5 total, 3 up, 5 in 2023-11-16T13:32:47.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:46.813166+0000 mon.smithi151 (mon.0) 406 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:47.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:46.813473+0000 mon.smithi151 (mon.0) 407 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:47.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:46.816536+0000 mon.smithi151 (mon.0) 408 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:47.026913+0000 mgr.smithi151.fcujra (mgr.14164) 127 : cephadm [INF] Detected new or changed devices on smithi151 2023-11-16T13:32:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:47.033960+0000 mon.smithi151 (mon.0) 409 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:47.034851+0000 mon.smithi151 (mon.0) 410 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:47.035408+0000 mon.smithi151 (mon.0) 411 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:47.035937+0000 mon.smithi151 (mon.0) 412 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:47.036445+0000 mon.smithi151 (mon.0) 413 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:47.036869+0000 mgr.smithi151.fcujra (mgr.14164) 128 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 3224M 2023-11-16T13:32:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:47.042001+0000 mon.smithi151 (mon.0) 414 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:48.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:47.045314+0000 mon.smithi151 (mon.0) 415 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:48.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:47.047989+0000 mon.smithi151 (mon.0) 416 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:48.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:47.048745+0000 mon.smithi151 (mon.0) 417 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:48.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:47.049214+0000 mon.smithi151 (mon.0) 418 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:48.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:47.225223+0000 mon.smithi167 (mon.1) 3 : audit [DBG] from='client.? 172.21.15.167:0/3549180368' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:32:48.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:47.814938+0000 mon.smithi151 (mon.0) 419 : cluster [INF] osd.3 [v2:172.21.15.151:6826/3194408863,v1:172.21.15.151:6827/3194408863] boot 2023-11-16T13:32:48.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:47.814970+0000 mon.smithi151 (mon.0) 420 : cluster [DBG] osdmap e26: 5 total, 4 up, 5 in 2023-11-16T13:32:48.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:47.815216+0000 mon.smithi151 (mon.0) 421 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:48.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:47.815403+0000 mon.smithi151 (mon.0) 422 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:48.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:47.026913+0000 mgr.smithi151.fcujra (mgr.14164) 127 : cephadm [INF] Detected new or changed devices on smithi151 2023-11-16T13:32:48.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:47.033960+0000 mon.smithi151 (mon.0) 409 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:48.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:47.034851+0000 mon.smithi151 (mon.0) 410 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:47.035408+0000 mon.smithi151 (mon.0) 411 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:47.035937+0000 mon.smithi151 (mon.0) 412 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:47.036445+0000 mon.smithi151 (mon.0) 413 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:47.036869+0000 mgr.smithi151.fcujra (mgr.14164) 128 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 3224M 2023-11-16T13:32:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:47.042001+0000 mon.smithi151 (mon.0) 414 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:47.045314+0000 mon.smithi151 (mon.0) 415 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:47.047989+0000 mon.smithi151 (mon.0) 416 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:47.048745+0000 mon.smithi151 (mon.0) 417 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:47.049214+0000 mon.smithi151 (mon.0) 418 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:47.225223+0000 mon.smithi167 (mon.1) 3 : audit [DBG] from='client.? 172.21.15.167:0/3549180368' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:32:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:47.814938+0000 mon.smithi151 (mon.0) 419 : cluster [INF] osd.3 [v2:172.21.15.151:6826/3194408863,v1:172.21.15.151:6827/3194408863] boot 2023-11-16T13:32:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:47.814970+0000 mon.smithi151 (mon.0) 420 : cluster [DBG] osdmap e26: 5 total, 4 up, 5 in 2023-11-16T13:32:48.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:47.815216+0000 mon.smithi151 (mon.0) 421 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:32:48.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:47.815403+0000 mon.smithi151 (mon.0) 422 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:49.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:46.090510+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:32:49.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:46.090599+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:32:49.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:48.118099+0000 mgr.smithi151.fcujra (mgr.14164) 129 : cluster [DBG] pgmap v88: 1 pgs: 1 creating+peering; 0 B data, 19 MiB used, 358 GiB / 358 GiB avail 2023-11-16T13:32:49.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:46.090510+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:32:49.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:46.090599+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:32:49.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:48.118099+0000 mgr.smithi151.fcujra (mgr.14164) 129 : cluster [DBG] pgmap v88: 1 pgs: 1 creating+peering; 0 B data, 19 MiB used, 358 GiB / 358 GiB avail 2023-11-16T13:32:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:49.040434+0000 mon.smithi151 (mon.0) 423 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-11-16T13:32:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:49.040664+0000 mon.smithi151 (mon.0) 424 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:49.513730+0000 mon.smithi151 (mon.0) 425 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:49.758948+0000 mon.smithi151 (mon.0) 426 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:50.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:49.040434+0000 mon.smithi151 (mon.0) 423 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-11-16T13:32:50.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:49.040664+0000 mon.smithi151 (mon.0) 424 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:50.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:49.513730+0000 mon.smithi151 (mon.0) 425 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:50.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:49.758948+0000 mon.smithi151 (mon.0) 426 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:52.090 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:51 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:50.118453+0000 mgr.smithi151.fcujra (mgr.14164) 130 : cluster [DBG] pgmap v90: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-11-16T13:32:52.090 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:51 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:51.392747+0000 mon.smithi151 (mon.0) 427 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-11-16T13:32:52.090 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:51 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:51.393464+0000 mon.smithi151 (mon.0) 428 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:52.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:51 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:50.118453+0000 mgr.smithi151.fcujra (mgr.14164) 130 : cluster [DBG] pgmap v90: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-11-16T13:32:52.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:51 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:51.392747+0000 mon.smithi151 (mon.0) 427 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-11-16T13:32:52.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:51 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:51.393464+0000 mon.smithi151 (mon.0) 428 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:53.016 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:51.394022+0000 mgr.smithi151.fcujra (mgr.14164) 131 : cephadm [INF] Deploying daemon osd.4 on smithi167 2023-11-16T13:32:53.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:51.394022+0000 mgr.smithi151.fcujra (mgr.14164) 131 : cephadm [INF] Deploying daemon osd.4 on smithi167 2023-11-16T13:32:53.689 INFO:teuthology.orchestra.run.smithi167.stdout:Created osd(s) 4 on host 'smithi167' 2023-11-16T13:32:54.016 DEBUG:teuthology.orchestra.run.smithi167:osd.4> sudo journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.4.service 2023-11-16T13:32:54.018 INFO:tasks.cephadm:Deploying osd.5 on smithi167 with /dev/vg_nvme/lv_3... 2023-11-16T13:32:54.018 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-11-16T13:32:54.049 INFO:journalctl@ceph.osd.4.smithi167.stdout:-- Logs begin at Thu 2023-11-16 13:20:04 UTC. -- 2023-11-16T13:32:54.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:52.118894+0000 mgr.smithi151.fcujra (mgr.14164) 132 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-11-16T13:32:54.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:53.677939+0000 mon.smithi151 (mon.0) 429 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:54.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:53.681129+0000 mon.smithi151 (mon.0) 430 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:54.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:53.681369+0000 mon.smithi151 (mon.0) 431 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:54.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:53.682881+0000 mon.smithi151 (mon.0) 432 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:54.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:53.685524+0000 mon.smithi151 (mon.0) 433 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:54.201 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:52.118894+0000 mgr.smithi151.fcujra (mgr.14164) 132 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-11-16T13:32:54.201 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:53.677939+0000 mon.smithi151 (mon.0) 429 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:54.201 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:53.681129+0000 mon.smithi151 (mon.0) 430 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:54.201 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:53.681369+0000 mon.smithi151 (mon.0) 431 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:54.201 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:53.682881+0000 mon.smithi151 (mon.0) 432 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:54.201 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:53.685524+0000 mon.smithi151 (mon.0) 433 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:54.507 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:32:54 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.4[115342]: debug 2023-11-16T13:32:54.142+0000 7f252c844080 -1 Falling back to public interface 2023-11-16T13:32:55.492 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-11-16T13:32:55.492 INFO:teuthology.orchestra.run.smithi167.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-11-16T13:32:55.492 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10+0 records in 2023-11-16T13:32:55.493 INFO:teuthology.orchestra.run.smithi167.stderr:10+0 records out 2023-11-16T13:32:55.493 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0122276 s, 858 MB/s 2023-11-16T13:32:55.493 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping successful for: 2023-11-16T13:32:55.918 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch daemon add osd smithi167:vg_nvme/lv_3 2023-11-16T13:32:55.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:55 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:54.119185+0000 mgr.smithi151.fcujra (mgr.14164) 133 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-11-16T13:32:55.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:55 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:55.264123+0000 mon.smithi151 (mon.0) 434 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:56.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:55 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:54.119185+0000 mgr.smithi151.fcujra (mgr.14164) 133 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-11-16T13:32:56.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:55 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:55.264123+0000 mon.smithi151 (mon.0) 434 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:57.034 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:32:56 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.4[115342]: debug 2023-11-16T13:32:56.908+0000 7f252c844080 -1 osd.4 0 log_to_monitors {default=true} 2023-11-16T13:32:57.332 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:56.071764+0000 mon.smithi151 (mon.0) 435 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:57.332 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:56.119589+0000 mgr.smithi151.fcujra (mgr.14164) 134 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-11-16T13:32:57.332 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:56.912160+0000 mon.smithi167 (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.167:6800/2155142958,v1:172.21.15.167:6801/2155142958]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-11-16T13:32:57.332 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:56.912627+0000 mon.smithi151 (mon.0) 436 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-11-16T13:32:57.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:56.071764+0000 mon.smithi151 (mon.0) 435 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:57.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:56.119589+0000 mgr.smithi151.fcujra (mgr.14164) 134 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-11-16T13:32:57.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:56.912160+0000 mon.smithi167 (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.167:6800/2155142958,v1:172.21.15.167:6801/2155142958]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-11-16T13:32:57.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:56.912627+0000 mon.smithi151 (mon.0) 436 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-11-16T13:32:58.272 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.4[115342]: debug 2023-11-16T13:32:58.085+0000 7f2522da7700 -1 osd.4 0 waiting for initial osdmap 2023-11-16T13:32:58.273 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.4[115342]: debug 2023-11-16T13:32:58.089+0000 7f251cd1e700 -1 osd.4 29 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2023-11-16T13:32:58.273 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.081270+0000 mon.smithi151 (mon.0) 437 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-11-16T13:32:58.273 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:57.081362+0000 mon.smithi151 (mon.0) 438 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-11-16T13:32:58.273 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.081737+0000 mon.smithi151 (mon.0) 439 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:58.273 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.083716+0000 mon.smithi167 (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.167:6800/2155142958,v1:172.21.15.167:6801/2155142958]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:32:58.273 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.084090+0000 mon.smithi151 (mon.0) 440 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:32:58.273 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.195165+0000 mgr.smithi151.fcujra (mgr.14164) 135 : audit [DBG] from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:32:58.273 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.195948+0000 mon.smithi151 (mon.0) 441 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:32:58.273 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.197543+0000 mon.smithi151 (mon.0) 442 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:32:58.274 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.198248+0000 mon.smithi151 (mon.0) 443 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:58.274 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.985372+0000 mon.smithi151 (mon.0) 444 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:58.274 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.986201+0000 mon.smithi151 (mon.0) 445 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:58.274 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.990517+0000 mon.smithi151 (mon.0) 446 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:58.274 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.994815+0000 mon.smithi151 (mon.0) 447 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:58.274 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.998439+0000 mon.smithi151 (mon.0) 448 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:58.274 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:57.999394+0000 mon.smithi151 (mon.0) 449 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:58.274 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:58.000006+0000 mon.smithi151 (mon.0) 450 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:58.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.081270+0000 mon.smithi151 (mon.0) 437 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-11-16T13:32:58.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:57.081362+0000 mon.smithi151 (mon.0) 438 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-11-16T13:32:58.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.081737+0000 mon.smithi151 (mon.0) 439 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:58.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.083716+0000 mon.smithi167 (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.167:6800/2155142958,v1:172.21.15.167:6801/2155142958]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:32:58.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.084090+0000 mon.smithi151 (mon.0) 440 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:32:58.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.195165+0000 mgr.smithi151.fcujra (mgr.14164) 135 : audit [DBG] from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:32:58.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.195948+0000 mon.smithi151 (mon.0) 441 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:32:58.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.197543+0000 mon.smithi151 (mon.0) 442 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:32:58.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.198248+0000 mon.smithi151 (mon.0) 443 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:58.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.985372+0000 mon.smithi151 (mon.0) 444 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:58.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.986201+0000 mon.smithi151 (mon.0) 445 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:32:58.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.990517+0000 mon.smithi151 (mon.0) 446 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:58.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.994815+0000 mon.smithi151 (mon.0) 447 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:32:58.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.998439+0000 mon.smithi151 (mon.0) 448 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:32:58.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:57.999394+0000 mon.smithi151 (mon.0) 449 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:32:58.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:58.000006+0000 mon.smithi151 (mon.0) 450 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:32:59.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:57.980397+0000 mgr.smithi151.fcujra (mgr.14164) 136 : cephadm [INF] Detected new or changed devices on smithi167 2023-11-16T13:32:59.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:32:57.986810+0000 mgr.smithi151.fcujra (mgr.14164) 137 : cephadm [INF] Adjusting osd_memory_target on smithi167 to 15970M 2023-11-16T13:32:59.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:58.082389+0000 mon.smithi151 (mon.0) 451 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-11-16T13:32:59.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:58.082455+0000 mon.smithi151 (mon.0) 452 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-11-16T13:32:59.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:58.082777+0000 mon.smithi151 (mon.0) 453 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:59.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:58.086087+0000 mon.smithi151 (mon.0) 454 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:59.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:58.119908+0000 mgr.smithi151.fcujra (mgr.14164) 138 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-11-16T13:32:59.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:58.936414+0000 mon.smithi167 (mon.1) 6 : audit [INF] from='client.? 172.21.15.167:0/3440358742' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b64a42c5-2048-40c6-b404-8e833e5a988e"}]: dispatch 2023-11-16T13:32:59.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:58.936909+0000 mon.smithi151 (mon.0) 455 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b64a42c5-2048-40c6-b404-8e833e5a988e"}]: dispatch 2023-11-16T13:32:59.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:58.943298+0000 mon.smithi151 (mon.0) 456 : cluster [INF] osd.4 [v2:172.21.15.167:6800/2155142958,v1:172.21.15.167:6801/2155142958] boot 2023-11-16T13:32:59.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:58.943367+0000 mon.smithi151 (mon.0) 457 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b64a42c5-2048-40c6-b404-8e833e5a988e"}]': finished 2023-11-16T13:32:59.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:58.943426+0000 mon.smithi151 (mon.0) 458 : cluster [DBG] osdmap e30: 6 total, 5 up, 6 in 2023-11-16T13:32:59.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:58.943553+0000 mon.smithi151 (mon.0) 459 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:59.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:32:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:58.943968+0000 mon.smithi151 (mon.0) 460 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:32:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:57.980397+0000 mgr.smithi151.fcujra (mgr.14164) 136 : cephadm [INF] Detected new or changed devices on smithi167 2023-11-16T13:32:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:32:57.986810+0000 mgr.smithi151.fcujra (mgr.14164) 137 : cephadm [INF] Adjusting osd_memory_target on smithi167 to 15970M 2023-11-16T13:32:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:58.082389+0000 mon.smithi151 (mon.0) 451 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-11-16T13:32:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:58.082455+0000 mon.smithi151 (mon.0) 452 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-11-16T13:32:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:58.082777+0000 mon.smithi151 (mon.0) 453 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:58.086087+0000 mon.smithi151 (mon.0) 454 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:58.119908+0000 mgr.smithi151.fcujra (mgr.14164) 138 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-11-16T13:32:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:58.936414+0000 mon.smithi167 (mon.1) 6 : audit [INF] from='client.? 172.21.15.167:0/3440358742' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b64a42c5-2048-40c6-b404-8e833e5a988e"}]: dispatch 2023-11-16T13:32:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:58.936909+0000 mon.smithi151 (mon.0) 455 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b64a42c5-2048-40c6-b404-8e833e5a988e"}]: dispatch 2023-11-16T13:32:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:58.943298+0000 mon.smithi151 (mon.0) 456 : cluster [INF] osd.4 [v2:172.21.15.167:6800/2155142958,v1:172.21.15.167:6801/2155142958] boot 2023-11-16T13:32:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:58.943367+0000 mon.smithi151 (mon.0) 457 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b64a42c5-2048-40c6-b404-8e833e5a988e"}]': finished 2023-11-16T13:32:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:58.943426+0000 mon.smithi151 (mon.0) 458 : cluster [DBG] osdmap e30: 6 total, 5 up, 6 in 2023-11-16T13:32:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:58.943553+0000 mon.smithi151 (mon.0) 459 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:32:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:32:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:58.943968+0000 mon.smithi151 (mon.0) 460 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:33:00.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:57.895485+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:33:00.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:57.895568+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:33:00.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:59.351912+0000 mon.smithi167 (mon.1) 7 : audit [DBG] from='client.? 172.21.15.167:0/3002894430' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:33:00.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:32:59.941782+0000 mon.smithi151 (mon.0) 461 : cluster [DBG] osdmap e31: 6 total, 5 up, 6 in 2023-11-16T13:33:00.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:32:59.941911+0000 mon.smithi151 (mon.0) 462 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:33:00.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:57.895485+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:33:00.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:57.895568+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:33:00.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:59.351912+0000 mon.smithi167 (mon.1) 7 : audit [DBG] from='client.? 172.21.15.167:0/3002894430' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:33:00.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:32:59.941782+0000 mon.smithi151 (mon.0) 461 : cluster [DBG] osdmap e31: 6 total, 5 up, 6 in 2023-11-16T13:33:00.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:32:59.941911+0000 mon.smithi151 (mon.0) 462 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:33:01.594 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:00.120169+0000 mgr.smithi151.fcujra (mgr.14164) 139 : cluster [DBG] pgmap v99: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-11-16T13:33:01.595 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:00.294577+0000 mon.smithi151 (mon.0) 463 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:01.595 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:00.299546+0000 mon.smithi151 (mon.0) 464 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:01.595 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:00.947086+0000 mon.smithi151 (mon.0) 465 : cluster [DBG] osdmap e32: 6 total, 5 up, 6 in 2023-11-16T13:33:01.595 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:00.947239+0000 mon.smithi151 (mon.0) 466 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:33:01.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:00.120169+0000 mgr.smithi151.fcujra (mgr.14164) 139 : cluster [DBG] pgmap v99: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-11-16T13:33:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:00.294577+0000 mon.smithi151 (mon.0) 463 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:00.299546+0000 mon.smithi151 (mon.0) 464 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:00.947086+0000 mon.smithi151 (mon.0) 465 : cluster [DBG] osdmap e32: 6 total, 5 up, 6 in 2023-11-16T13:33:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:00.947239+0000 mon.smithi151 (mon.0) 466 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:33:03.484 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:02.120612+0000 mgr.smithi151.fcujra (mgr.14164) 140 : cluster [DBG] pgmap v101: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-11-16T13:33:03.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:02.120612+0000 mgr.smithi151.fcujra (mgr.14164) 140 : cluster [DBG] pgmap v101: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-11-16T13:33:04.508 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:03.550954+0000 mon.smithi151 (mon.0) 467 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-11-16T13:33:04.508 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:03.551902+0000 mon.smithi151 (mon.0) 468 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:04.508 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:33:03.552601+0000 mgr.smithi151.fcujra (mgr.14164) 141 : cephadm [INF] Deploying daemon osd.5 on smithi167 2023-11-16T13:33:04.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:03.550954+0000 mon.smithi151 (mon.0) 467 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-11-16T13:33:04.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:03.551902+0000 mon.smithi151 (mon.0) 468 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:04.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:33:03.552601+0000 mgr.smithi151.fcujra (mgr.14164) 141 : cephadm [INF] Deploying daemon osd.5 on smithi167 2023-11-16T13:33:05.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:04.121018+0000 mgr.smithi151.fcujra (mgr.14164) 142 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-11-16T13:33:05.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:04.121018+0000 mgr.smithi151.fcujra (mgr.14164) 142 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-11-16T13:33:05.764 INFO:teuthology.orchestra.run.smithi167.stdout:Created osd(s) 5 on host 'smithi167' 2023-11-16T13:33:06.091 DEBUG:teuthology.orchestra.run.smithi167:osd.5> sudo journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.5.service 2023-11-16T13:33:06.093 INFO:tasks.cephadm:Deploying osd.6 on smithi167 with /dev/vg_nvme/lv_2... 2023-11-16T13:33:06.093 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-11-16T13:33:06.126 INFO:journalctl@ceph.osd.5.smithi167.stdout:-- Logs begin at Thu 2023-11-16 13:20:04 UTC. -- 2023-11-16T13:33:06.308 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:33:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.5[118177]: debug 2023-11-16T13:33:06.224+0000 7f082b89e080 -1 Falling back to public interface 2023-11-16T13:33:06.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:05.751264+0000 mon.smithi151 (mon.0) 469 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:33:06.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:05.753259+0000 mon.smithi151 (mon.0) 470 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:06.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:05.755865+0000 mon.smithi151 (mon.0) 471 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:06.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:05.758029+0000 mon.smithi151 (mon.0) 472 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:33:06.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:05.760957+0000 mon.smithi151 (mon.0) 473 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:06.606 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:06 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:05.751264+0000 mon.smithi151 (mon.0) 469 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:33:06.606 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:06 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:05.753259+0000 mon.smithi151 (mon.0) 470 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:06.606 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:06 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:05.755865+0000 mon.smithi151 (mon.0) 471 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:06.607 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:06 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:05.758029+0000 mon.smithi151 (mon.0) 472 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:33:06.607 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:06 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:05.760957+0000 mon.smithi151 (mon.0) 473 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:07.599 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-11-16T13:33:07.599 INFO:teuthology.orchestra.run.smithi167.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-11-16T13:33:07.599 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10+0 records in 2023-11-16T13:33:07.599 INFO:teuthology.orchestra.run.smithi167.stderr:10+0 records out 2023-11-16T13:33:07.599 INFO:teuthology.orchestra.run.smithi167.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0153232 s, 684 MB/s 2023-11-16T13:33:07.599 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping successful for: 2023-11-16T13:33:07.916 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:06.121483+0000 mgr.smithi151.fcujra (mgr.14164) 143 : cluster [DBG] pgmap v103: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-11-16T13:33:07.916 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:07.432930+0000 mon.smithi151 (mon.0) 474 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:08.087 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch daemon add osd smithi167:vg_nvme/lv_2 2023-11-16T13:33:08.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:06.121483+0000 mgr.smithi151.fcujra (mgr.14164) 143 : cluster [DBG] pgmap v103: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-11-16T13:33:08.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:07.432930+0000 mon.smithi151 (mon.0) 474 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:09.346 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:33:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.5[118177]: debug 2023-11-16T13:33:08.967+0000 7f082b89e080 -1 osd.5 0 log_to_monitors {default=true} 2023-11-16T13:33:09.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:08.121750+0000 mgr.smithi151.fcujra (mgr.14164) 144 : cluster [DBG] pgmap v104: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-11-16T13:33:09.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:08.125136+0000 mon.smithi151 (mon.0) 475 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:09.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:08.973181+0000 mon.smithi151 (mon.0) 476 : audit [INF] from='osd.5 [v2:172.21.15.167:6808/1283627108,v1:172.21.15.167:6809/1283627108]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-11-16T13:33:09.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:08.121750+0000 mgr.smithi151.fcujra (mgr.14164) 144 : cluster [DBG] pgmap v104: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-11-16T13:33:09.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:08.125136+0000 mon.smithi151 (mon.0) 475 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:09.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:08.973181+0000 mon.smithi151 (mon.0) 476 : audit [INF] from='osd.5 [v2:172.21.15.167:6808/1283627108,v1:172.21.15.167:6809/1283627108]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-11-16T13:33:10.171 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:09.401220+0000 mgr.smithi151.fcujra (mgr.14164) 145 : audit [DBG] from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:33:10.171 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:09.402118+0000 mon.smithi151 (mon.0) 477 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:33:10.171 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:09.403583+0000 mon.smithi151 (mon.0) 478 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:33:10.171 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:09.404274+0000 mon.smithi151 (mon.0) 479 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:10.171 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:09.886159+0000 mon.smithi151 (mon.0) 480 : audit [INF] from='osd.5 [v2:172.21.15.167:6808/1283627108,v1:172.21.15.167:6809/1283627108]' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-11-16T13:33:10.171 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:09.886215+0000 mon.smithi151 (mon.0) 481 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-11-16T13:33:10.171 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:09.886337+0000 mon.smithi151 (mon.0) 482 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:33:10.171 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:09.886726+0000 mon.smithi151 (mon.0) 483 : audit [INF] from='osd.5 [v2:172.21.15.167:6808/1283627108,v1:172.21.15.167:6809/1283627108]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:33:10.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:09.401220+0000 mgr.smithi151.fcujra (mgr.14164) 145 : audit [DBG] from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:33:10.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:09.402118+0000 mon.smithi151 (mon.0) 477 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:33:10.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:09.403583+0000 mon.smithi151 (mon.0) 478 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:33:10.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:09.404274+0000 mon.smithi151 (mon.0) 479 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:10.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:09.886159+0000 mon.smithi151 (mon.0) 480 : audit [INF] from='osd.5 [v2:172.21.15.167:6808/1283627108,v1:172.21.15.167:6809/1283627108]' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-11-16T13:33:10.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:09.886215+0000 mon.smithi151 (mon.0) 481 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-11-16T13:33:10.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:09.886337+0000 mon.smithi151 (mon.0) 482 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:33:10.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:09.886726+0000 mon.smithi151 (mon.0) 483 : audit [INF] from='osd.5 [v2:172.21.15.167:6808/1283627108,v1:172.21.15.167:6809/1283627108]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:33:11.161 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:33:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.5[118177]: debug 2023-11-16T13:33:10.889+0000 7f0821e01700 -1 osd.5 0 waiting for initial osdmap 2023-11-16T13:33:11.161 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:33:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.5[118177]: debug 2023-11-16T13:33:10.894+0000 7f081a575700 -1 osd.5 34 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2023-11-16T13:33:11.442 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:10.122206+0000 mgr.smithi151.fcujra (mgr.14164) 146 : cluster [DBG] pgmap v106: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-11-16T13:33:11.442 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:33:10.155539+0000 mgr.smithi151.fcujra (mgr.14164) 147 : cephadm [INF] Detected new or changed devices on smithi167 2023-11-16T13:33:11.443 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:10.160285+0000 mon.smithi151 (mon.0) 484 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:11.443 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:10.161189+0000 mon.smithi151 (mon.0) 485 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:11.443 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:10.161961+0000 mon.smithi151 (mon.0) 486 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:11.443 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:33:10.162497+0000 mgr.smithi151.fcujra (mgr.14164) 148 : cephadm [INF] Adjusting osd_memory_target on smithi167 to 7985M 2023-11-16T13:33:11.443 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:10.166110+0000 mon.smithi151 (mon.0) 487 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:11.443 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:10.169725+0000 mon.smithi151 (mon.0) 488 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:11.443 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:10.173258+0000 mon.smithi151 (mon.0) 489 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:33:11.443 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:10.174324+0000 mon.smithi151 (mon.0) 490 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:11.443 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:10.175024+0000 mon.smithi151 (mon.0) 491 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:33:11.443 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:10.888281+0000 mon.smithi151 (mon.0) 492 : audit [INF] from='osd.5 [v2:172.21.15.167:6808/1283627108,v1:172.21.15.167:6809/1283627108]' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-11-16T13:33:11.443 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:10.888344+0000 mon.smithi151 (mon.0) 493 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-11-16T13:33:11.443 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:10.888443+0000 mon.smithi151 (mon.0) 494 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:33:11.444 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:10.891438+0000 mon.smithi151 (mon.0) 495 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:33:11.444 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:11.149197+0000 mon.smithi167 (mon.1) 8 : audit [INF] from='client.? 172.21.15.167:0/4075999455' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "56d60c4c-9997-4096-abbc-8605a9d12a09"}]: dispatch 2023-11-16T13:33:11.444 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:11.149666+0000 mon.smithi151 (mon.0) 496 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "56d60c4c-9997-4096-abbc-8605a9d12a09"}]: dispatch 2023-11-16T13:33:11.444 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:11.155374+0000 mon.smithi151 (mon.0) 497 : cluster [INF] osd.5 [v2:172.21.15.167:6808/1283627108,v1:172.21.15.167:6809/1283627108] boot 2023-11-16T13:33:11.444 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:11.155438+0000 mon.smithi151 (mon.0) 498 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "56d60c4c-9997-4096-abbc-8605a9d12a09"}]': finished 2023-11-16T13:33:11.444 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:11.155503+0000 mon.smithi151 (mon.0) 499 : cluster [DBG] osdmap e35: 7 total, 6 up, 7 in 2023-11-16T13:33:11.444 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:11.155786+0000 mon.smithi151 (mon.0) 500 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:33:11.444 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:11.156043+0000 mon.smithi151 (mon.0) 501 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:11.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:10.122206+0000 mgr.smithi151.fcujra (mgr.14164) 146 : cluster [DBG] pgmap v106: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-11-16T13:33:11.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:33:10.155539+0000 mgr.smithi151.fcujra (mgr.14164) 147 : cephadm [INF] Detected new or changed devices on smithi167 2023-11-16T13:33:11.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:10.160285+0000 mon.smithi151 (mon.0) 484 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:11.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:10.161189+0000 mon.smithi151 (mon.0) 485 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:11.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:10.161961+0000 mon.smithi151 (mon.0) 486 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:11.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:33:10.162497+0000 mgr.smithi151.fcujra (mgr.14164) 148 : cephadm [INF] Adjusting osd_memory_target on smithi167 to 7985M 2023-11-16T13:33:11.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:10.166110+0000 mon.smithi151 (mon.0) 487 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:11.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:10.169725+0000 mon.smithi151 (mon.0) 488 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:11.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:10.173258+0000 mon.smithi151 (mon.0) 489 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:33:11.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:10.174324+0000 mon.smithi151 (mon.0) 490 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:11.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:10.175024+0000 mon.smithi151 (mon.0) 491 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:33:11.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:10.888281+0000 mon.smithi151 (mon.0) 492 : audit [INF] from='osd.5 [v2:172.21.15.167:6808/1283627108,v1:172.21.15.167:6809/1283627108]' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-11-16T13:33:11.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:10.888344+0000 mon.smithi151 (mon.0) 493 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-11-16T13:33:11.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:10.888443+0000 mon.smithi151 (mon.0) 494 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:33:11.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:10.891438+0000 mon.smithi151 (mon.0) 495 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:33:11.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:11.149197+0000 mon.smithi167 (mon.1) 8 : audit [INF] from='client.? 172.21.15.167:0/4075999455' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "56d60c4c-9997-4096-abbc-8605a9d12a09"}]: dispatch 2023-11-16T13:33:11.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:11.149666+0000 mon.smithi151 (mon.0) 496 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "56d60c4c-9997-4096-abbc-8605a9d12a09"}]: dispatch 2023-11-16T13:33:11.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:11.155374+0000 mon.smithi151 (mon.0) 497 : cluster [INF] osd.5 [v2:172.21.15.167:6808/1283627108,v1:172.21.15.167:6809/1283627108] boot 2023-11-16T13:33:11.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:11.155438+0000 mon.smithi151 (mon.0) 498 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "56d60c4c-9997-4096-abbc-8605a9d12a09"}]': finished 2023-11-16T13:33:11.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:11.155503+0000 mon.smithi151 (mon.0) 499 : cluster [DBG] osdmap e35: 7 total, 6 up, 7 in 2023-11-16T13:33:11.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:11.155786+0000 mon.smithi151 (mon.0) 500 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:33:11.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:11.156043+0000 mon.smithi151 (mon.0) 501 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:12.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:10.010362+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:33:12.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:10.010457+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:33:12.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:11.572049+0000 mon.smithi167 (mon.1) 9 : audit [DBG] from='client.? 172.21.15.167:0/1671446352' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:33:12.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:12.155207+0000 mon.smithi151 (mon.0) 502 : cluster [DBG] osdmap e36: 7 total, 6 up, 7 in 2023-11-16T13:33:12.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:12.155316+0000 mon.smithi151 (mon.0) 503 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:12.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:10.010362+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:33:12.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:10.010457+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:33:12.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:11.572049+0000 mon.smithi167 (mon.1) 9 : audit [DBG] from='client.? 172.21.15.167:0/1671446352' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:33:12.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:12.155207+0000 mon.smithi151 (mon.0) 502 : cluster [DBG] osdmap e36: 7 total, 6 up, 7 in 2023-11-16T13:33:12.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:12.155316+0000 mon.smithi151 (mon.0) 503 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:13.962 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:12.122688+0000 mgr.smithi151.fcujra (mgr.14164) 149 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-11-16T13:33:13.962 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:12.598632+0000 mon.smithi151 (mon.0) 504 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:13.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:12.601960+0000 mon.smithi151 (mon.0) 505 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:13.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:13.159625+0000 mon.smithi151 (mon.0) 506 : cluster [DBG] osdmap e37: 7 total, 6 up, 7 in 2023-11-16T13:33:13.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:13.159762+0000 mon.smithi151 (mon.0) 507 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:13.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:12.122688+0000 mgr.smithi151.fcujra (mgr.14164) 149 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-11-16T13:33:13.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:12.598632+0000 mon.smithi151 (mon.0) 504 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:13.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:12.601960+0000 mon.smithi151 (mon.0) 505 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:13.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:13.159625+0000 mon.smithi151 (mon.0) 506 : cluster [DBG] osdmap e37: 7 total, 6 up, 7 in 2023-11-16T13:33:13.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:13.159762+0000 mon.smithi151 (mon.0) 507 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:15.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:14.123093+0000 mgr.smithi151.fcujra (mgr.14164) 150 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-11-16T13:33:15.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:14.123093+0000 mgr.smithi151.fcujra (mgr.14164) 150 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-11-16T13:33:16.750 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:15.782708+0000 mon.smithi151 (mon.0) 508 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-11-16T13:33:16.750 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:15.783466+0000 mon.smithi151 (mon.0) 509 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:16.750 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:33:15.784062+0000 mgr.smithi151.fcujra (mgr.14164) 151 : cephadm [INF] Deploying daemon osd.6 on smithi167 2023-11-16T13:33:16.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:15.782708+0000 mon.smithi151 (mon.0) 508 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-11-16T13:33:16.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:15.783466+0000 mon.smithi151 (mon.0) 509 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:16.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:33:15.784062+0000 mgr.smithi151.fcujra (mgr.14164) 151 : cephadm [INF] Deploying daemon osd.6 on smithi167 2023-11-16T13:33:18.005 INFO:teuthology.orchestra.run.smithi167.stdout:Created osd(s) 6 on host 'smithi167' 2023-11-16T13:33:18.153 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:16.123397+0000 mgr.smithi151.fcujra (mgr.14164) 152 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-11-16T13:33:18.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:16.123397+0000 mgr.smithi151.fcujra (mgr.14164) 152 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-11-16T13:33:18.334 DEBUG:teuthology.orchestra.run.smithi167:osd.6> sudo journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.6.service 2023-11-16T13:33:18.336 INFO:tasks.cephadm:Deploying osd.7 on smithi167 with /dev/vg_nvme/lv_1... 2023-11-16T13:33:18.336 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-11-16T13:33:18.366 INFO:journalctl@ceph.osd.6.smithi167.stdout:-- Logs begin at Thu 2023-11-16 13:20:04 UTC. -- 2023-11-16T13:33:18.726 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:33:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.6[121036]: debug 2023-11-16T13:33:18.464+0000 7f89badc4080 -1 Falling back to public interface 2023-11-16T13:33:19.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:17.990717+0000 mon.smithi151 (mon.0) 510 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:33:19.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:17.992502+0000 mon.smithi151 (mon.0) 511 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:19.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:17.993287+0000 mon.smithi151 (mon.0) 512 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:33:19.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:17.996935+0000 mon.smithi151 (mon.0) 513 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:19.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:18.001690+0000 mon.smithi151 (mon.0) 514 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:19.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:17.990717+0000 mon.smithi151 (mon.0) 510 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:33:19.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:17.992502+0000 mon.smithi151 (mon.0) 511 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:19.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:17.993287+0000 mon.smithi151 (mon.0) 512 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:33:19.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:17.996935+0000 mon.smithi151 (mon.0) 513 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:19.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:18.001690+0000 mon.smithi151 (mon.0) 514 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:20.057 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-11-16T13:33:20.057 INFO:teuthology.orchestra.run.smithi167.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-11-16T13:33:20.057 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10+0 records in 2023-11-16T13:33:20.057 INFO:teuthology.orchestra.run.smithi167.stderr:10+0 records out 2023-11-16T13:33:20.057 INFO:teuthology.orchestra.run.smithi167.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0123482 s, 849 MB/s 2023-11-16T13:33:20.057 INFO:teuthology.orchestra.run.smithi167.stderr:--> Zapping successful for: 2023-11-16T13:33:20.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:18.123884+0000 mgr.smithi151.fcujra (mgr.14164) 153 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-11-16T13:33:20.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:19.643286+0000 mon.smithi151 (mon.0) 515 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:20.261 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:18.123884+0000 mgr.smithi151.fcujra (mgr.14164) 153 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-11-16T13:33:20.261 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:19.643286+0000 mon.smithi151 (mon.0) 515 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:20.618 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch daemon add osd smithi167:vg_nvme/lv_1 2023-11-16T13:33:21.346 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:33:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.6[121036]: debug 2023-11-16T13:33:21.195+0000 7f89badc4080 -1 osd.6 0 log_to_monitors {default=true} 2023-11-16T13:33:22.153 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:20.124250+0000 mgr.smithi151.fcujra (mgr.14164) 154 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-11-16T13:33:22.154 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:21.199452+0000 mon.smithi167 (mon.1) 10 : audit [INF] from='osd.6 [v2:172.21.15.167:6816/3288345058,v1:172.21.15.167:6817/3288345058]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-11-16T13:33:22.154 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:21.199865+0000 mon.smithi151 (mon.0) 516 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-11-16T13:33:22.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:20.124250+0000 mgr.smithi151.fcujra (mgr.14164) 154 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-11-16T13:33:22.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:21.199452+0000 mon.smithi167 (mon.1) 10 : audit [INF] from='osd.6 [v2:172.21.15.167:6816/3288345058,v1:172.21.15.167:6817/3288345058]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-11-16T13:33:22.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:21.199865+0000 mon.smithi151 (mon.0) 516 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-11-16T13:33:22.962 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.6[121036]: debug 2023-11-16T13:33:22.898+0000 7f89b1327700 -1 osd.6 0 waiting for initial osdmap 2023-11-16T13:33:22.962 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.6[121036]: debug 2023-11-16T13:33:22.905+0000 7f89aba9f700 -1 osd.6 39 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2023-11-16T13:33:22.962 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:21.897501+0000 mon.smithi151 (mon.0) 517 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-11-16T13:33:22.962 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:21.897567+0000 mon.smithi151 (mon.0) 518 : cluster [DBG] osdmap e38: 7 total, 6 up, 7 in 2023-11-16T13:33:22.962 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:21.898008+0000 mon.smithi151 (mon.0) 519 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:22.962 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:21.898102+0000 mon.smithi167 (mon.1) 11 : audit [INF] from='osd.6 [v2:172.21.15.167:6816/3288345058,v1:172.21.15.167:6817/3288345058]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:33:22.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:21.898403+0000 mon.smithi151 (mon.0) 520 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:33:22.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:21.997603+0000 mon.smithi151 (mon.0) 521 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:33:22.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:21.999160+0000 mon.smithi151 (mon.0) 522 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:33:22.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:21.999911+0000 mon.smithi151 (mon.0) 523 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:22.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.245706+0000 mon.smithi151 (mon.0) 524 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:33:22.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.245985+0000 mon.smithi151 (mon.0) 525 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:33:22.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.314866+0000 mon.smithi151 (mon.0) 526 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:22.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.315843+0000 mon.smithi151 (mon.0) 527 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:22.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.316620+0000 mon.smithi151 (mon.0) 528 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:22.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.317381+0000 mon.smithi151 (mon.0) 529 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:22.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.321533+0000 mon.smithi151 (mon.0) 530 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:22.963 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.324794+0000 mon.smithi151 (mon.0) 531 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:22.964 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.328338+0000 mon.smithi151 (mon.0) 532 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:33:22.964 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.329328+0000 mon.smithi151 (mon.0) 533 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:22.964 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.329981+0000 mon.smithi151 (mon.0) 534 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:33:22.964 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.333720+0000 mon.smithi151 (mon.0) 535 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:22.964 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.896454+0000 mon.smithi151 (mon.0) 536 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-11-16T13:33:22.964 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:22.896551+0000 mon.smithi151 (mon.0) 537 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-11-16T13:33:22.964 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.896717+0000 mon.smithi151 (mon.0) 538 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:22.964 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:22.899930+0000 mon.smithi151 (mon.0) 539 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:21.897501+0000 mon.smithi151 (mon.0) 517 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-11-16T13:33:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:21.897567+0000 mon.smithi151 (mon.0) 518 : cluster [DBG] osdmap e38: 7 total, 6 up, 7 in 2023-11-16T13:33:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:21.898008+0000 mon.smithi151 (mon.0) 519 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:21.898102+0000 mon.smithi167 (mon.1) 11 : audit [INF] from='osd.6 [v2:172.21.15.167:6816/3288345058,v1:172.21.15.167:6817/3288345058]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:33:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:21.898403+0000 mon.smithi151 (mon.0) 520 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:33:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:21.997603+0000 mon.smithi151 (mon.0) 521 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-16T13:33:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:21.999160+0000 mon.smithi151 (mon.0) 522 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-16T13:33:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:21.999911+0000 mon.smithi151 (mon.0) 523 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.245706+0000 mon.smithi151 (mon.0) 524 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:33:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.245985+0000 mon.smithi151 (mon.0) 525 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:33:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.314866+0000 mon.smithi151 (mon.0) 526 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.315843+0000 mon.smithi151 (mon.0) 527 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.316620+0000 mon.smithi151 (mon.0) 528 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.317381+0000 mon.smithi151 (mon.0) 529 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.321533+0000 mon.smithi151 (mon.0) 530 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.324794+0000 mon.smithi151 (mon.0) 531 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.328338+0000 mon.smithi151 (mon.0) 532 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:33:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.329328+0000 mon.smithi151 (mon.0) 533 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.329981+0000 mon.smithi151 (mon.0) 534 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:33:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.333720+0000 mon.smithi151 (mon.0) 535 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:23.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.896454+0000 mon.smithi151 (mon.0) 536 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-11-16T13:33:23.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:22.896551+0000 mon.smithi151 (mon.0) 537 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-11-16T13:33:23.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.896717+0000 mon.smithi151 (mon.0) 538 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:23.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:22.899930+0000 mon.smithi151 (mon.0) 539 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:21.996668+0000 mgr.smithi151.fcujra (mgr.14164) 155 : audit [DBG] from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:33:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:22.124727+0000 mgr.smithi151.fcujra (mgr.14164) 156 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-11-16T13:33:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:33:22.309978+0000 mgr.smithi151.fcujra (mgr.14164) 157 : cephadm [INF] Detected new or changed devices on smithi167 2023-11-16T13:33:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:33:22.318019+0000 mgr.smithi151.fcujra (mgr.14164) 158 : cephadm [INF] Adjusting osd_memory_target on smithi167 to 5323M 2023-11-16T13:33:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:23.603545+0000 mon.smithi167 (mon.1) 12 : audit [INF] from='client.? 172.21.15.167:0/1711096084' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1003d4e3-4e43-400e-b820-e8ce80a64450"}]: dispatch 2023-11-16T13:33:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:23.604269+0000 mon.smithi151 (mon.0) 540 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1003d4e3-4e43-400e-b820-e8ce80a64450"}]: dispatch 2023-11-16T13:33:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:23.610295+0000 mon.smithi151 (mon.0) 541 : cluster [INF] osd.6 [v2:172.21.15.167:6816/3288345058,v1:172.21.15.167:6817/3288345058] boot 2023-11-16T13:33:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:23.610397+0000 mon.smithi151 (mon.0) 542 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1003d4e3-4e43-400e-b820-e8ce80a64450"}]': finished 2023-11-16T13:33:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:23.610486+0000 mon.smithi151 (mon.0) 543 : cluster [DBG] osdmap e40: 8 total, 7 up, 8 in 2023-11-16T13:33:24.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:23.610621+0000 mon.smithi151 (mon.0) 544 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:24.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:23.611207+0000 mon.smithi151 (mon.0) 545 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:21.996668+0000 mgr.smithi151.fcujra (mgr.14164) 155 : audit [DBG] from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi167:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:33:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:22.124727+0000 mgr.smithi151.fcujra (mgr.14164) 156 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-11-16T13:33:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:33:22.309978+0000 mgr.smithi151.fcujra (mgr.14164) 157 : cephadm [INF] Detected new or changed devices on smithi167 2023-11-16T13:33:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:33:22.318019+0000 mgr.smithi151.fcujra (mgr.14164) 158 : cephadm [INF] Adjusting osd_memory_target on smithi167 to 5323M 2023-11-16T13:33:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:23.603545+0000 mon.smithi167 (mon.1) 12 : audit [INF] from='client.? 172.21.15.167:0/1711096084' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1003d4e3-4e43-400e-b820-e8ce80a64450"}]: dispatch 2023-11-16T13:33:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:23.604269+0000 mon.smithi151 (mon.0) 540 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1003d4e3-4e43-400e-b820-e8ce80a64450"}]: dispatch 2023-11-16T13:33:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:23.610295+0000 mon.smithi151 (mon.0) 541 : cluster [INF] osd.6 [v2:172.21.15.167:6816/3288345058,v1:172.21.15.167:6817/3288345058] boot 2023-11-16T13:33:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:23.610397+0000 mon.smithi151 (mon.0) 542 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1003d4e3-4e43-400e-b820-e8ce80a64450"}]': finished 2023-11-16T13:33:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:23.610486+0000 mon.smithi151 (mon.0) 543 : cluster [DBG] osdmap e40: 8 total, 7 up, 8 in 2023-11-16T13:33:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:23.610621+0000 mon.smithi151 (mon.0) 544 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:33:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:23.611207+0000 mon.smithi151 (mon.0) 545 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:25.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:22.193157+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:33:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:22.193260+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:33:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:24.012723+0000 mon.smithi167 (mon.1) 13 : audit [DBG] from='client.? 172.21.15.167:0/830121892' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:33:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:24.611023+0000 mon.smithi151 (mon.0) 546 : cluster [DBG] osdmap e41: 8 total, 7 up, 8 in 2023-11-16T13:33:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:24.611218+0000 mon.smithi151 (mon.0) 547 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:22.193157+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:33:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:22.193260+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:33:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:24.012723+0000 mon.smithi167 (mon.1) 13 : audit [DBG] from='client.? 172.21.15.167:0/830121892' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-16T13:33:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:24.611023+0000 mon.smithi151 (mon.0) 546 : cluster [DBG] osdmap e41: 8 total, 7 up, 8 in 2023-11-16T13:33:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:24.611218+0000 mon.smithi151 (mon.0) 547 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:26.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:25 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:24.125231+0000 mgr.smithi151.fcujra (mgr.14164) 159 : cluster [DBG] pgmap v120: 1 pgs: 1 peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:26.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:25 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:25.612141+0000 mon.smithi151 (mon.0) 548 : cluster [DBG] osdmap e42: 8 total, 7 up, 8 in 2023-11-16T13:33:26.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:25 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:25.612317+0000 mon.smithi151 (mon.0) 549 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:26.242 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:25 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:24.125231+0000 mgr.smithi151.fcujra (mgr.14164) 159 : cluster [DBG] pgmap v120: 1 pgs: 1 peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:26.242 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:25 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:25.612141+0000 mon.smithi151 (mon.0) 548 : cluster [DBG] osdmap e42: 8 total, 7 up, 8 in 2023-11-16T13:33:26.242 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:25 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:25.612317+0000 mon.smithi151 (mon.0) 549 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:27.193 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:26.125696+0000 mgr.smithi151.fcujra (mgr.14164) 160 : cluster [DBG] pgmap v123: 1 pgs: 1 peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:27.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:26.125696+0000 mgr.smithi151.fcujra (mgr.14164) 160 : cluster [DBG] pgmap v123: 1 pgs: 1 peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:29.329 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:28.126073+0000 mgr.smithi151.fcujra (mgr.14164) 161 : cluster [DBG] pgmap v124: 1 pgs: 1 peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:29.330 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:28.323308+0000 mon.smithi151 (mon.0) 550 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-11-16T13:33:29.330 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:28.325001+0000 mon.smithi151 (mon.0) 551 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:29.330 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:33:28.325717+0000 mgr.smithi151.fcujra (mgr.14164) 162 : cephadm [INF] Deploying daemon osd.7 on smithi167 2023-11-16T13:33:29.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:28.126073+0000 mgr.smithi151.fcujra (mgr.14164) 161 : cluster [DBG] pgmap v124: 1 pgs: 1 peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:29.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:28.323308+0000 mon.smithi151 (mon.0) 550 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-11-16T13:33:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:28.325001+0000 mon.smithi151 (mon.0) 551 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:33:28.325717+0000 mgr.smithi151.fcujra (mgr.14164) 162 : cephadm [INF] Deploying daemon osd.7 on smithi167 2023-11-16T13:33:30.587 INFO:teuthology.orchestra.run.smithi167.stdout:Created osd(s) 7 on host 'smithi167' 2023-11-16T13:33:30.907 DEBUG:teuthology.orchestra.run.smithi167:osd.7> sudo journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.7.service 2023-11-16T13:33:30.909 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-11-16T13:33:30.909 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd stat -f json 2023-11-16T13:33:30.944 INFO:journalctl@ceph.osd.7.smithi167.stdout:-- Logs begin at Thu 2023-11-16 13:20:04 UTC. -- 2023-11-16T13:33:31.183 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:33:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.7[123906]: debug 2023-11-16T13:33:31.052+0000 7f4109ce2080 -1 Falling back to public interface 2023-11-16T13:33:31.213 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:30.126398+0000 mgr.smithi151.fcujra (mgr.14164) 163 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:31.214 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:30.572661+0000 mon.smithi151 (mon.0) 552 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:33:31.214 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:30.574597+0000 mon.smithi151 (mon.0) 553 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:31.214 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:30.576970+0000 mon.smithi151 (mon.0) 554 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:31.214 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:30.579232+0000 mon.smithi151 (mon.0) 555 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:33:31.214 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:30.582485+0000 mon.smithi151 (mon.0) 556 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:31.589 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:30.126398+0000 mgr.smithi151.fcujra (mgr.14164) 163 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:31.589 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:30.572661+0000 mon.smithi151 (mon.0) 552 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:33:31.589 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:30.574597+0000 mon.smithi151 (mon.0) 553 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:31.589 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:30.576970+0000 mon.smithi151 (mon.0) 554 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:31.589 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:30.579232+0000 mon.smithi151 (mon.0) 555 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:33:31.589 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:30.582485+0000 mon.smithi151 (mon.0) 556 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:32.223 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:33:32.575 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":42,"num_osds":8,"num_up_osds":7,"osd_up_since":1700141603,"num_in_osds":8,"osd_in_since":1700141603,"num_remapped_pgs":0} 2023-11-16T13:33:33.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:31.951258+0000 mon.smithi151 (mon.0) 557 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:33.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:32.126713+0000 mgr.smithi151.fcujra (mgr.14164) 164 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:33.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:32.222059+0000 mon.smithi151 (mon.0) 558 : audit [DBG] from='client.? 172.21.15.151:0/2159292946' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-11-16T13:33:33.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:31.951258+0000 mon.smithi151 (mon.0) 557 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:33.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:32.126713+0000 mgr.smithi151.fcujra (mgr.14164) 164 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:33.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:32.222059+0000 mon.smithi151 (mon.0) 558 : audit [DBG] from='client.? 172.21.15.151:0/2159292946' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-11-16T13:33:33.577 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd stat -f json 2023-11-16T13:33:33.958 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:33:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.7[123906]: debug 2023-11-16T13:33:33.801+0000 7f4109ce2080 -1 osd.7 0 log_to_monitors {default=true} 2023-11-16T13:33:34.188 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:33.804733+0000 mon.smithi167 (mon.1) 14 : audit [INF] from='osd.7 [v2:172.21.15.167:6824/2815469433,v1:172.21.15.167:6825/2815469433]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-11-16T13:33:34.188 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:33.805127+0000 mon.smithi151 (mon.0) 559 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-11-16T13:33:34.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:33.804733+0000 mon.smithi167 (mon.1) 14 : audit [INF] from='osd.7 [v2:172.21.15.167:6824/2815469433,v1:172.21.15.167:6825/2815469433]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-11-16T13:33:34.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:33.805127+0000 mon.smithi151 (mon.0) 559 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-11-16T13:33:34.861 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:33:35.224 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:33.960941+0000 mon.smithi151 (mon.0) 560 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-11-16T13:33:35.224 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:33.961022+0000 mon.smithi151 (mon.0) 561 : cluster [DBG] osdmap e43: 8 total, 7 up, 8 in 2023-11-16T13:33:35.224 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:33.961143+0000 mon.smithi151 (mon.0) 562 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:35.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:33.961955+0000 mon.smithi167 (mon.1) 15 : audit [INF] from='osd.7 [v2:172.21.15.167:6824/2815469433,v1:172.21.15.167:6825/2815469433]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:33:35.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:33.962252+0000 mon.smithi151 (mon.0) 563 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:33:35.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:34.127057+0000 mgr.smithi151.fcujra (mgr.14164) 165 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:35.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:33:34.236443+0000 mgr.smithi151.fcujra (mgr.14164) 166 : cephadm [INF] Detected new or changed devices on smithi167 2023-11-16T13:33:35.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.241023+0000 mon.smithi151 (mon.0) 564 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:35.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.484266+0000 mon.smithi151 (mon.0) 565 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:35.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.484999+0000 mon.smithi151 (mon.0) 566 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:35.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.485559+0000 mon.smithi151 (mon.0) 567 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:35.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.486127+0000 mon.smithi151 (mon.0) 568 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:35.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:33:34.486503+0000 mgr.smithi151.fcujra (mgr.14164) 167 : cephadm [INF] Adjusting osd_memory_target on smithi167 to 3992M 2023-11-16T13:33:35.225 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.490365+0000 mon.smithi151 (mon.0) 569 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:35.226 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.494128+0000 mon.smithi151 (mon.0) 570 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:35.226 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.497062+0000 mon.smithi151 (mon.0) 571 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:33:35.226 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.497874+0000 mon.smithi151 (mon.0) 572 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:35.226 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.498393+0000 mon.smithi151 (mon.0) 573 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:33:35.226 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.504127+0000 mon.smithi151 (mon.0) 574 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:35.226 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.860577+0000 mon.smithi151 (mon.0) 575 : audit [DBG] from='client.? 172.21.15.151:0/2681630955' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-11-16T13:33:35.227 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":43,"num_osds":8,"num_up_osds":7,"osd_up_since":1700141603,"num_in_osds":8,"osd_in_since":1700141603,"num_remapped_pgs":0} 2023-11-16T13:33:35.346 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.7[123906]: debug 2023-11-16T13:33:34.966+0000 7f4100245700 -1 osd.7 0 waiting for initial osdmap 2023-11-16T13:33:35.346 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.7[123906]: debug 2023-11-16T13:33:34.973+0000 7f40fb9bf700 -1 osd.7 44 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2023-11-16T13:33:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:33.960941+0000 mon.smithi151 (mon.0) 560 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-11-16T13:33:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:33.961022+0000 mon.smithi151 (mon.0) 561 : cluster [DBG] osdmap e43: 8 total, 7 up, 8 in 2023-11-16T13:33:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:33.961143+0000 mon.smithi151 (mon.0) 562 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:33.961955+0000 mon.smithi167 (mon.1) 15 : audit [INF] from='osd.7 [v2:172.21.15.167:6824/2815469433,v1:172.21.15.167:6825/2815469433]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:33:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:33.962252+0000 mon.smithi151 (mon.0) 563 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:33:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:34.127057+0000 mgr.smithi151.fcujra (mgr.14164) 165 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:33:34.236443+0000 mgr.smithi151.fcujra (mgr.14164) 166 : cephadm [INF] Detected new or changed devices on smithi167 2023-11-16T13:33:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.241023+0000 mon.smithi151 (mon.0) 564 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.484266+0000 mon.smithi151 (mon.0) 565 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.484999+0000 mon.smithi151 (mon.0) 566 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.485559+0000 mon.smithi151 (mon.0) 567 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.486127+0000 mon.smithi151 (mon.0) 568 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:33:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:33:34.486503+0000 mgr.smithi151.fcujra (mgr.14164) 167 : cephadm [INF] Adjusting osd_memory_target on smithi167 to 3992M 2023-11-16T13:33:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.490365+0000 mon.smithi151 (mon.0) 569 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.494128+0000 mon.smithi151 (mon.0) 570 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.497062+0000 mon.smithi151 (mon.0) 571 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:33:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.497874+0000 mon.smithi151 (mon.0) 572 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:33:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.498393+0000 mon.smithi151 (mon.0) 573 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:33:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.504127+0000 mon.smithi151 (mon.0) 574 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:33:35.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.860577+0000 mon.smithi151 (mon.0) 575 : audit [DBG] from='client.? 172.21.15.151:0/2681630955' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-11-16T13:33:36.228 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd stat -f json 2023-11-16T13:33:36.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.963279+0000 mon.smithi151 (mon.0) 576 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-11-16T13:33:36.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:34.963354+0000 mon.smithi151 (mon.0) 577 : cluster [DBG] osdmap e44: 8 total, 7 up, 8 in 2023-11-16T13:33:36.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.963670+0000 mon.smithi151 (mon.0) 578 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:36.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:34.967481+0000 mon.smithi151 (mon.0) 579 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.963279+0000 mon.smithi151 (mon.0) 576 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]': finished 2023-11-16T13:33:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:34.963354+0000 mon.smithi151 (mon.0) 577 : cluster [DBG] osdmap e44: 8 total, 7 up, 8 in 2023-11-16T13:33:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.963670+0000 mon.smithi151 (mon.0) 578 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:34.967481+0000 mon.smithi151 (mon.0) 579 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:37.157 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:34.825404+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:33:37.157 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:34.825555+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:33:37.158 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:35.969006+0000 mon.smithi151 (mon.0) 580 : cluster [INF] osd.7 [v2:172.21.15.167:6824/2815469433,v1:172.21.15.167:6825/2815469433] boot 2023-11-16T13:33:37.158 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:35.969069+0000 mon.smithi151 (mon.0) 581 : cluster [DBG] osdmap e45: 8 total, 8 up, 8 in 2023-11-16T13:33:37.158 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:35.969423+0000 mon.smithi151 (mon.0) 582 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:37.158 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:36.127371+0000 mgr.smithi151.fcujra (mgr.14164) 168 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:37.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:34.825404+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-11-16T13:33:37.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:34.825555+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-11-16T13:33:37.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:35.969006+0000 mon.smithi151 (mon.0) 580 : cluster [INF] osd.7 [v2:172.21.15.167:6824/2815469433,v1:172.21.15.167:6825/2815469433] boot 2023-11-16T13:33:37.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:35.969069+0000 mon.smithi151 (mon.0) 581 : cluster [DBG] osdmap e45: 8 total, 8 up, 8 in 2023-11-16T13:33:37.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:35.969423+0000 mon.smithi151 (mon.0) 582 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:33:37.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:36.127371+0000 mgr.smithi151.fcujra (mgr.14164) 168 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-11-16T13:33:37.502 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:33:37.849 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":46,"num_osds":8,"num_up_osds":8,"osd_up_since":1700141615,"num_in_osds":8,"osd_in_since":1700141603,"num_remapped_pgs":0} 2023-11-16T13:33:37.849 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd dump --format=json 2023-11-16T13:33:38.113 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:38.210 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:36.983370+0000 mon.smithi151 (mon.0) 583 : cluster [DBG] osdmap e46: 8 total, 8 up, 8 in 2023-11-16T13:33:38.211 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:37.501803+0000 mon.smithi151 (mon.0) 584 : audit [DBG] from='client.? 172.21.15.151:0/3639950507' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-11-16T13:33:38.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:36.983370+0000 mon.smithi151 (mon.0) 583 : cluster [DBG] osdmap e46: 8 total, 8 up, 8 in 2023-11-16T13:33:38.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:37.501803+0000 mon.smithi151 (mon.0) 584 : audit [DBG] from='client.? 172.21.15.151:0/3639950507' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-11-16T13:33:39.127 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:33:39.127 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":46,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","created":"2023-11-16T13:29:32.864321+0000","modified":"2023-11-16T13:33:36.970525+0000","last_up_change":"2023-11-16T13:33:35.964694+0000","last_in_change":"2023-11-16T13:33:23.604904+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-11-16T13:32:42.207015+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"d0acfa3f-92df-4f58-8a5e-3578a88f486c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":3541700964},{"type":"v1","addr":"172.21.15.151:6803","nonce":3541700964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":3541700964},{"type":"v1","addr":"172.21.15.151:6805","nonce":3541700964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":3541700964},{"type":"v1","addr":"172.21.15.151:6809","nonce":3541700964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":3541700964},{"type":"v1","addr":"172.21.15.151:6807","nonce":3541700964}]},"public_addr":"172.21.15.151:6803/3541700964","cluster_addr":"172.21.15.151:6805/3541700964","heartbeat_back_addr":"172.21.15.151:6809/3541700964","heartbeat_front_addr":"172.21.15.151:6807/3541700964","state":["exists","up"]},{"osd":1,"uuid":"a9923417-ee98-4100-9df6-035b2a72db39","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":3233915009},{"type":"v1","addr":"172.21.15.151:6811","nonce":3233915009}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":3233915009},{"type":"v1","addr":"172.21.15.151:6813","nonce":3233915009}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":3233915009},{"type":"v1","addr":"172.21.15.151:6817","nonce":3233915009}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":3233915009},{"type":"v1","addr":"172.21.15.151:6815","nonce":3233915009}]},"public_addr":"172.21.15.151:6811/3233915009","cluster_addr":"172.21.15.151:6813/3233915009","heartbeat_back_addr":"172.21.15.151:6817/3233915009","heartbeat_front_addr":"172.21.15.151:6815/3233915009","state":["exists","up"]},{"osd":2,"uuid":"5b581b07-525f-4aca-97ce-089a9cec4901","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":1911452143},{"type":"v1","addr":"172.21.15.151:6819","nonce":1911452143}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":1911452143},{"type":"v1","addr":"172.21.15.151:6821","nonce":1911452143}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":1911452143},{"type":"v1","addr":"172.21.15.151:6825","nonce":1911452143}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":1911452143},{"type":"v1","addr":"172.21.15.151:6823","nonce":1911452143}]},"public_addr":"172.21.15.151:6819/1911452143","cluster_addr":"172.21.15.151:6821/1911452143","heartbeat_back_addr":"172.21.15.151:6825/1911452143","heartbeat_front_addr":"172.21.15.151:6823/1911452143","state":["exists","up"]},{"osd":3,"uuid":"ac93c609-8473-40fc-9dc5-bea25b31d2c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":3194408863},{"type":"v1","addr":"172.21.15.151:6827","nonce":3194408863}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":3194408863},{"type":"v1","addr":"172.21.15.151:6829","nonce":3194408863}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6832","nonce":3194408863},{"type":"v1","addr":"172.21.15.151:6833","nonce":3194408863}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":3194408863},{"type":"v1","addr":"172.21.15.151:6831","nonce":3194408863}]},"public_addr":"172.21.15.151:6827/3194408863","cluster_addr":"172.21.15.151:6829/3194408863","heartbeat_back_addr":"172.21.15.151:6833/3194408863","heartbeat_front_addr":"172.21.15.151:6831/3194408863","state":["exists","up"]},{"osd":4,"uuid":"b187f517-360e-4ded-9afe-b6ca35c88906","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":2155142958},{"type":"v1","addr":"172.21.15.167:6801","nonce":2155142958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":2155142958},{"type":"v1","addr":"172.21.15.167:6803","nonce":2155142958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":2155142958},{"type":"v1","addr":"172.21.15.167:6807","nonce":2155142958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":2155142958},{"type":"v1","addr":"172.21.15.167:6805","nonce":2155142958}]},"public_addr":"172.21.15.167:6801/2155142958","cluster_addr":"172.21.15.167:6803/2155142958","heartbeat_back_addr":"172.21.15.167:6807/2155142958","heartbeat_front_addr":"172.21.15.167:6805/2155142958","state":["exists","up"]},{"osd":5,"uuid":"b64a42c5-2048-40c6-b404-8e833e5a988e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":35,"up_thru":36,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":1283627108},{"type":"v1","addr":"172.21.15.167:6809","nonce":1283627108}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":1283627108},{"type":"v1","addr":"172.21.15.167:6811","nonce":1283627108}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":1283627108},{"type":"v1","addr":"172.21.15.167:6815","nonce":1283627108}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":1283627108},{"type":"v1","addr":"172.21.15.167:6813","nonce":1283627108}]},"public_addr":"172.21.15.167:6809/1283627108","cluster_addr":"172.21.15.167:6811/1283627108","heartbeat_back_addr":"172.21.15.167:6815/1283627108","heartbeat_front_addr":"172.21.15.167:6813/1283627108","state":["exists","up"]},{"osd":6,"uuid":"56d60c4c-9997-4096-abbc-8605a9d12a09","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":3288345058},{"type":"v1","addr":"172.21.15.167:6817","nonce":3288345058}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":3288345058},{"type":"v1","addr":"172.21.15.167:6819","nonce":3288345058}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":3288345058},{"type":"v1","addr":"172.21.15.167:6823","nonce":3288345058}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":3288345058},{"type":"v1","addr":"172.21.15.167:6821","nonce":3288345058}]},"public_addr":"172.21.15.167:6817/3288345058","cluster_addr":"172.21.15.167:6819/3288345058","heartbeat_back_addr":"172.21.15.167:6823/3288345058","heartbeat_front_addr":"172.21.15.167:6821/3288345058","state":["exists","up"]},{"osd":7,"uuid":"1003d4e3-4e43-400e-b820-e8ce80a64450","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6824","nonce":2815469433},{"type":"v1","addr":"172.21.15.167:6825","nonce":2815469433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6826","nonce":2815469433},{"type":"v1","addr":"172.21.15.167:6827","nonce":2815469433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6830","nonce":2815469433},{"type":"v1","addr":"172.21.15.167:6831","nonce":2815469433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6828","nonce":2815469433},{"type":"v1","addr":"172.21.15.167:6829","nonce":2815469433}]},"public_addr":"172.21.15.167:6825/2815469433","cluster_addr":"172.21.15.167:6827/2815469433","heartbeat_back_addr":"172.21.15.167:6831/2815469433","heartbeat_front_addr":"172.21.15.167:6829/2815469433","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:00.207404+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:16.452550+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:32.525724+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:46.090602+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:57.895571+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:33:10.010459+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:33:22.193263+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:33:34.825557+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.151:0/1188402741":"2023-11-17T13:30:22.036975+0000","172.21.15.151:6800/1453319453":"2023-11-17T13:30:22.036975+0000","172.21.15.151:0/4190395784":"2023-11-17T13:30:10.106477+0000","172.21.15.151:0/1947503251":"2023-11-17T13:30:22.036975+0000","172.21.15.151:6801/1453319453":"2023-11-17T13:30:22.036975+0000","172.21.15.151:6801/4217080091":"2023-11-17T13:30:10.106477+0000","172.21.15.151:6800/4217080091":"2023-11-17T13:30:10.106477+0000","172.21.15.151:0/1874828182":"2023-11-17T13:30:10.106477+0000","172.21.15.151:6800/3823821624":"2023-11-17T13:29:52.404698+0000","172.21.15.151:6801/3823821624":"2023-11-17T13:29:52.404698+0000","172.21.15.151:0/1542985892":"2023-11-17T13:29:52.404698+0000","172.21.15.151:0/3861002685":"2023-11-17T13:29:52.404698+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-11-16T13:33:39.152 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:38.127691+0000 mgr.smithi151.fcujra (mgr.14164) 169 : cluster [DBG] pgmap v133: 1 pgs: 1 peering; 0 B data, 41 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:39.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:38.127691+0000 mgr.smithi151.fcujra (mgr.14164) 169 : cluster [DBG] pgmap v133: 1 pgs: 1 peering; 0 B data, 41 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:39.492 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-11-16T13:32:42.207015+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-11-16T13:33:39.492 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd pool get device_health_metrics pg_num 2023-11-16T13:33:39.713 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:40.087 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:39 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:39.124395+0000 mon.smithi151 (mon.0) 585 : audit [DBG] from='client.? 172.21.15.151:0/1636239384' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-11-16T13:33:40.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:39 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:39.124395+0000 mon.smithi151 (mon.0) 585 : audit [DBG] from='client.? 172.21.15.151:0/1636239384' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-11-16T13:33:40.787 INFO:teuthology.orchestra.run.smithi151.stdout:pg_num: 1 2023-11-16T13:33:41.147 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:40.128076+0000 mgr.smithi151.fcujra (mgr.14164) 170 : cluster [DBG] pgmap v134: 1 pgs: 1 peering; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:41.147 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:40.785988+0000 mon.smithi151 (mon.0) 586 : audit [DBG] from='client.? 172.21.15.151:0/613501895' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-11-16T13:33:41.149 INFO:tasks.cephadm:Setting up client nodes... 2023-11-16T13:33:41.149 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-11-16T13:33:41.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:40.128076+0000 mgr.smithi151.fcujra (mgr.14164) 170 : cluster [DBG] pgmap v134: 1 pgs: 1 peering; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:41.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:40.785988+0000 mon.smithi151 (mon.0) 586 : audit [DBG] from='client.? 172.21.15.151:0/613501895' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-11-16T13:33:42.474 INFO:teuthology.orchestra.run.smithi151.stdout:[client.0] 2023-11-16T13:33:42.475 INFO:teuthology.orchestra.run.smithi151.stdout: key = AQA2GlZl1BkGHBAA7U8/iQRnV6mF+dsvMs+V7Q== 2023-11-16T13:33:42.840 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-11-16T13:33:42.840 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-11-16T13:33:42.840 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-11-16T13:33:42.881 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-11-16T13:33:42.881 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-11-16T13:33:42.881 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph mgr dump --format=json 2023-11-16T13:33:43.142 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:43.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:42.128570+0000 mgr.smithi151.fcujra (mgr.14164) 171 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:43.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:42.470012+0000 mon.smithi151 (mon.0) 587 : audit [INF] from='client.? 172.21.15.151:0/1676327083' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-11-16T13:33:43.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:42.473496+0000 mon.smithi151 (mon.0) 588 : audit [INF] from='client.? 172.21.15.151:0/1676327083' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-11-16T13:33:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:42.128570+0000 mgr.smithi151.fcujra (mgr.14164) 171 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:42.470012+0000 mon.smithi151 (mon.0) 587 : audit [INF] from='client.? 172.21.15.151:0/1676327083' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-11-16T13:33:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:42.473496+0000 mon.smithi151 (mon.0) 588 : audit [INF] from='client.? 172.21.15.151:0/1676327083' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-11-16T13:33:44.292 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:33:44.657 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":18,"active_gid":14164,"active_name":"smithi151.fcujra","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":554453731},{"type":"v1","addr":"172.21.15.151:6801","nonce":554453731}]},"active_addr":"172.21.15.151:6801/554453731","active_change":"2023-11-16T13:30:22.037259+0000","active_mgr_features":4540138297136906239,"available":true,"standbys":[{"gid":14230,"name":"smithi167.xxnzsz","mgr_features":4540138297136906239,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"docker.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"docker.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Registry to which we should normalize unqualified image names","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_HOST":{"name":"RGW_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_PORT":{"name":"RGW_API_PORT","type":"str","level":"advanced","flags":0,"default_value":"80","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SCHEME":{"name":"RGW_API_SCHEME","type":"str","level":"advanced","flags":0,"default_value":"http","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_USER_ID":{"name":"RGW_API_USER_ID","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"persist_interval":{"name":"persist_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how frequently to persist completed events","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"docker.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"docker.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Registry to which we should normalize unqualified image names","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_HOST":{"name":"RGW_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_PORT":{"name":"RGW_API_PORT","type":"str","level":"advanced","flags":0,"default_value":"80","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SCHEME":{"name":"RGW_API_SCHEME","type":"str","level":"advanced","flags":0,"default_value":"http","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_USER_ID":{"name":"RGW_API_USER_ID","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"persist_interval":{"name":"persist_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how frequently to persist completed events","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.151:8443/","prometheus":"http://172.21.15.151:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":4,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.151:0","nonce":2980240956}]},{"addrvec":[{"type":"v2","addr":"172.21.15.151:0","nonce":2770688045}]}]}} 2023-11-16T13:33:44.662 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-11-16T13:33:44.662 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-11-16T13:33:44.662 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd dump --format=json 2023-11-16T13:33:44.885 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:45.336 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:44.128894+0000 mgr.smithi151.fcujra (mgr.14164) 172 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:45.336 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:44.286290+0000 mon.smithi151 (mon.0) 589 : audit [DBG] from='client.? 172.21.15.151:0/415151851' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-11-16T13:33:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:44.128894+0000 mgr.smithi151.fcujra (mgr.14164) 172 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:44.286290+0000 mon.smithi151 (mon.0) 589 : audit [DBG] from='client.? 172.21.15.151:0/415151851' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-11-16T13:33:45.976 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:33:45.977 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":46,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","created":"2023-11-16T13:29:32.864321+0000","modified":"2023-11-16T13:33:36.970525+0000","last_up_change":"2023-11-16T13:33:35.964694+0000","last_in_change":"2023-11-16T13:33:23.604904+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-11-16T13:32:42.207015+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"d0acfa3f-92df-4f58-8a5e-3578a88f486c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":3541700964},{"type":"v1","addr":"172.21.15.151:6803","nonce":3541700964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":3541700964},{"type":"v1","addr":"172.21.15.151:6805","nonce":3541700964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":3541700964},{"type":"v1","addr":"172.21.15.151:6809","nonce":3541700964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":3541700964},{"type":"v1","addr":"172.21.15.151:6807","nonce":3541700964}]},"public_addr":"172.21.15.151:6803/3541700964","cluster_addr":"172.21.15.151:6805/3541700964","heartbeat_back_addr":"172.21.15.151:6809/3541700964","heartbeat_front_addr":"172.21.15.151:6807/3541700964","state":["exists","up"]},{"osd":1,"uuid":"a9923417-ee98-4100-9df6-035b2a72db39","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":3233915009},{"type":"v1","addr":"172.21.15.151:6811","nonce":3233915009}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":3233915009},{"type":"v1","addr":"172.21.15.151:6813","nonce":3233915009}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":3233915009},{"type":"v1","addr":"172.21.15.151:6817","nonce":3233915009}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":3233915009},{"type":"v1","addr":"172.21.15.151:6815","nonce":3233915009}]},"public_addr":"172.21.15.151:6811/3233915009","cluster_addr":"172.21.15.151:6813/3233915009","heartbeat_back_addr":"172.21.15.151:6817/3233915009","heartbeat_front_addr":"172.21.15.151:6815/3233915009","state":["exists","up"]},{"osd":2,"uuid":"5b581b07-525f-4aca-97ce-089a9cec4901","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":1911452143},{"type":"v1","addr":"172.21.15.151:6819","nonce":1911452143}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":1911452143},{"type":"v1","addr":"172.21.15.151:6821","nonce":1911452143}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":1911452143},{"type":"v1","addr":"172.21.15.151:6825","nonce":1911452143}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":1911452143},{"type":"v1","addr":"172.21.15.151:6823","nonce":1911452143}]},"public_addr":"172.21.15.151:6819/1911452143","cluster_addr":"172.21.15.151:6821/1911452143","heartbeat_back_addr":"172.21.15.151:6825/1911452143","heartbeat_front_addr":"172.21.15.151:6823/1911452143","state":["exists","up"]},{"osd":3,"uuid":"ac93c609-8473-40fc-9dc5-bea25b31d2c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":3194408863},{"type":"v1","addr":"172.21.15.151:6827","nonce":3194408863}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":3194408863},{"type":"v1","addr":"172.21.15.151:6829","nonce":3194408863}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6832","nonce":3194408863},{"type":"v1","addr":"172.21.15.151:6833","nonce":3194408863}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":3194408863},{"type":"v1","addr":"172.21.15.151:6831","nonce":3194408863}]},"public_addr":"172.21.15.151:6827/3194408863","cluster_addr":"172.21.15.151:6829/3194408863","heartbeat_back_addr":"172.21.15.151:6833/3194408863","heartbeat_front_addr":"172.21.15.151:6831/3194408863","state":["exists","up"]},{"osd":4,"uuid":"b187f517-360e-4ded-9afe-b6ca35c88906","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":2155142958},{"type":"v1","addr":"172.21.15.167:6801","nonce":2155142958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":2155142958},{"type":"v1","addr":"172.21.15.167:6803","nonce":2155142958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":2155142958},{"type":"v1","addr":"172.21.15.167:6807","nonce":2155142958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":2155142958},{"type":"v1","addr":"172.21.15.167:6805","nonce":2155142958}]},"public_addr":"172.21.15.167:6801/2155142958","cluster_addr":"172.21.15.167:6803/2155142958","heartbeat_back_addr":"172.21.15.167:6807/2155142958","heartbeat_front_addr":"172.21.15.167:6805/2155142958","state":["exists","up"]},{"osd":5,"uuid":"b64a42c5-2048-40c6-b404-8e833e5a988e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":35,"up_thru":36,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":1283627108},{"type":"v1","addr":"172.21.15.167:6809","nonce":1283627108}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":1283627108},{"type":"v1","addr":"172.21.15.167:6811","nonce":1283627108}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":1283627108},{"type":"v1","addr":"172.21.15.167:6815","nonce":1283627108}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":1283627108},{"type":"v1","addr":"172.21.15.167:6813","nonce":1283627108}]},"public_addr":"172.21.15.167:6809/1283627108","cluster_addr":"172.21.15.167:6811/1283627108","heartbeat_back_addr":"172.21.15.167:6815/1283627108","heartbeat_front_addr":"172.21.15.167:6813/1283627108","state":["exists","up"]},{"osd":6,"uuid":"56d60c4c-9997-4096-abbc-8605a9d12a09","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":3288345058},{"type":"v1","addr":"172.21.15.167:6817","nonce":3288345058}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":3288345058},{"type":"v1","addr":"172.21.15.167:6819","nonce":3288345058}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":3288345058},{"type":"v1","addr":"172.21.15.167:6823","nonce":3288345058}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":3288345058},{"type":"v1","addr":"172.21.15.167:6821","nonce":3288345058}]},"public_addr":"172.21.15.167:6817/3288345058","cluster_addr":"172.21.15.167:6819/3288345058","heartbeat_back_addr":"172.21.15.167:6823/3288345058","heartbeat_front_addr":"172.21.15.167:6821/3288345058","state":["exists","up"]},{"osd":7,"uuid":"1003d4e3-4e43-400e-b820-e8ce80a64450","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6824","nonce":2815469433},{"type":"v1","addr":"172.21.15.167:6825","nonce":2815469433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6826","nonce":2815469433},{"type":"v1","addr":"172.21.15.167:6827","nonce":2815469433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6830","nonce":2815469433},{"type":"v1","addr":"172.21.15.167:6831","nonce":2815469433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6828","nonce":2815469433},{"type":"v1","addr":"172.21.15.167:6829","nonce":2815469433}]},"public_addr":"172.21.15.167:6825/2815469433","cluster_addr":"172.21.15.167:6827/2815469433","heartbeat_back_addr":"172.21.15.167:6831/2815469433","heartbeat_front_addr":"172.21.15.167:6829/2815469433","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:00.207404+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:16.452550+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:32.525724+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:46.090602+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:57.895571+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:33:10.010459+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:33:22.193263+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:33:34.825557+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.151:0/1188402741":"2023-11-17T13:30:22.036975+0000","172.21.15.151:6800/1453319453":"2023-11-17T13:30:22.036975+0000","172.21.15.151:0/4190395784":"2023-11-17T13:30:10.106477+0000","172.21.15.151:0/1947503251":"2023-11-17T13:30:22.036975+0000","172.21.15.151:6801/1453319453":"2023-11-17T13:30:22.036975+0000","172.21.15.151:6801/4217080091":"2023-11-17T13:30:10.106477+0000","172.21.15.151:6800/4217080091":"2023-11-17T13:30:10.106477+0000","172.21.15.151:0/1874828182":"2023-11-17T13:30:10.106477+0000","172.21.15.151:6800/3823821624":"2023-11-17T13:29:52.404698+0000","172.21.15.151:6801/3823821624":"2023-11-17T13:29:52.404698+0000","172.21.15.151:0/1542985892":"2023-11-17T13:29:52.404698+0000","172.21.15.151:0/3861002685":"2023-11-17T13:29:52.404698+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-11-16T13:33:46.356 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-11-16T13:33:46.357 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd dump --format=json 2023-11-16T13:33:46.578 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:46.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:45.974770+0000 mon.smithi151 (mon.0) 590 : audit [DBG] from='client.? 172.21.15.151:0/2920970083' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-11-16T13:33:47.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:45.974770+0000 mon.smithi151 (mon.0) 590 : audit [DBG] from='client.? 172.21.15.151:0/2920970083' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-11-16T13:33:47.637 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:33:47.637 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":46,"fsid":"11fdb452-8484-11ee-8dbe-212e2dc638e7","created":"2023-11-16T13:29:32.864321+0000","modified":"2023-11-16T13:33:36.970525+0000","last_up_change":"2023-11-16T13:33:35.964694+0000","last_in_change":"2023-11-16T13:33:23.604904+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-11-16T13:32:42.207015+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"d0acfa3f-92df-4f58-8a5e-3578a88f486c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":3541700964},{"type":"v1","addr":"172.21.15.151:6803","nonce":3541700964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":3541700964},{"type":"v1","addr":"172.21.15.151:6805","nonce":3541700964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":3541700964},{"type":"v1","addr":"172.21.15.151:6809","nonce":3541700964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":3541700964},{"type":"v1","addr":"172.21.15.151:6807","nonce":3541700964}]},"public_addr":"172.21.15.151:6803/3541700964","cluster_addr":"172.21.15.151:6805/3541700964","heartbeat_back_addr":"172.21.15.151:6809/3541700964","heartbeat_front_addr":"172.21.15.151:6807/3541700964","state":["exists","up"]},{"osd":1,"uuid":"a9923417-ee98-4100-9df6-035b2a72db39","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":3233915009},{"type":"v1","addr":"172.21.15.151:6811","nonce":3233915009}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":3233915009},{"type":"v1","addr":"172.21.15.151:6813","nonce":3233915009}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":3233915009},{"type":"v1","addr":"172.21.15.151:6817","nonce":3233915009}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":3233915009},{"type":"v1","addr":"172.21.15.151:6815","nonce":3233915009}]},"public_addr":"172.21.15.151:6811/3233915009","cluster_addr":"172.21.15.151:6813/3233915009","heartbeat_back_addr":"172.21.15.151:6817/3233915009","heartbeat_front_addr":"172.21.15.151:6815/3233915009","state":["exists","up"]},{"osd":2,"uuid":"5b581b07-525f-4aca-97ce-089a9cec4901","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":1911452143},{"type":"v1","addr":"172.21.15.151:6819","nonce":1911452143}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":1911452143},{"type":"v1","addr":"172.21.15.151:6821","nonce":1911452143}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":1911452143},{"type":"v1","addr":"172.21.15.151:6825","nonce":1911452143}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":1911452143},{"type":"v1","addr":"172.21.15.151:6823","nonce":1911452143}]},"public_addr":"172.21.15.151:6819/1911452143","cluster_addr":"172.21.15.151:6821/1911452143","heartbeat_back_addr":"172.21.15.151:6825/1911452143","heartbeat_front_addr":"172.21.15.151:6823/1911452143","state":["exists","up"]},{"osd":3,"uuid":"ac93c609-8473-40fc-9dc5-bea25b31d2c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":3194408863},{"type":"v1","addr":"172.21.15.151:6827","nonce":3194408863}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":3194408863},{"type":"v1","addr":"172.21.15.151:6829","nonce":3194408863}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6832","nonce":3194408863},{"type":"v1","addr":"172.21.15.151:6833","nonce":3194408863}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":3194408863},{"type":"v1","addr":"172.21.15.151:6831","nonce":3194408863}]},"public_addr":"172.21.15.151:6827/3194408863","cluster_addr":"172.21.15.151:6829/3194408863","heartbeat_back_addr":"172.21.15.151:6833/3194408863","heartbeat_front_addr":"172.21.15.151:6831/3194408863","state":["exists","up"]},{"osd":4,"uuid":"b187f517-360e-4ded-9afe-b6ca35c88906","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":2155142958},{"type":"v1","addr":"172.21.15.167:6801","nonce":2155142958}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":2155142958},{"type":"v1","addr":"172.21.15.167:6803","nonce":2155142958}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":2155142958},{"type":"v1","addr":"172.21.15.167:6807","nonce":2155142958}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":2155142958},{"type":"v1","addr":"172.21.15.167:6805","nonce":2155142958}]},"public_addr":"172.21.15.167:6801/2155142958","cluster_addr":"172.21.15.167:6803/2155142958","heartbeat_back_addr":"172.21.15.167:6807/2155142958","heartbeat_front_addr":"172.21.15.167:6805/2155142958","state":["exists","up"]},{"osd":5,"uuid":"b64a42c5-2048-40c6-b404-8e833e5a988e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":35,"up_thru":36,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":1283627108},{"type":"v1","addr":"172.21.15.167:6809","nonce":1283627108}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":1283627108},{"type":"v1","addr":"172.21.15.167:6811","nonce":1283627108}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":1283627108},{"type":"v1","addr":"172.21.15.167:6815","nonce":1283627108}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":1283627108},{"type":"v1","addr":"172.21.15.167:6813","nonce":1283627108}]},"public_addr":"172.21.15.167:6809/1283627108","cluster_addr":"172.21.15.167:6811/1283627108","heartbeat_back_addr":"172.21.15.167:6815/1283627108","heartbeat_front_addr":"172.21.15.167:6813/1283627108","state":["exists","up"]},{"osd":6,"uuid":"56d60c4c-9997-4096-abbc-8605a9d12a09","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":3288345058},{"type":"v1","addr":"172.21.15.167:6817","nonce":3288345058}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":3288345058},{"type":"v1","addr":"172.21.15.167:6819","nonce":3288345058}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":3288345058},{"type":"v1","addr":"172.21.15.167:6823","nonce":3288345058}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":3288345058},{"type":"v1","addr":"172.21.15.167:6821","nonce":3288345058}]},"public_addr":"172.21.15.167:6817/3288345058","cluster_addr":"172.21.15.167:6819/3288345058","heartbeat_back_addr":"172.21.15.167:6823/3288345058","heartbeat_front_addr":"172.21.15.167:6821/3288345058","state":["exists","up"]},{"osd":7,"uuid":"1003d4e3-4e43-400e-b820-e8ce80a64450","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6824","nonce":2815469433},{"type":"v1","addr":"172.21.15.167:6825","nonce":2815469433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6826","nonce":2815469433},{"type":"v1","addr":"172.21.15.167:6827","nonce":2815469433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6830","nonce":2815469433},{"type":"v1","addr":"172.21.15.167:6831","nonce":2815469433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6828","nonce":2815469433},{"type":"v1","addr":"172.21.15.167:6829","nonce":2815469433}]},"public_addr":"172.21.15.167:6825/2815469433","cluster_addr":"172.21.15.167:6827/2815469433","heartbeat_back_addr":"172.21.15.167:6831/2815469433","heartbeat_front_addr":"172.21.15.167:6829/2815469433","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:00.207404+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:16.452550+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:32.525724+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:46.090602+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:32:57.895571+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:33:10.010459+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:33:22.193263+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2023-11-16T13:33:34.825557+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.151:0/1188402741":"2023-11-17T13:30:22.036975+0000","172.21.15.151:6800/1453319453":"2023-11-17T13:30:22.036975+0000","172.21.15.151:0/4190395784":"2023-11-17T13:30:10.106477+0000","172.21.15.151:0/1947503251":"2023-11-17T13:30:22.036975+0000","172.21.15.151:6801/1453319453":"2023-11-17T13:30:22.036975+0000","172.21.15.151:6801/4217080091":"2023-11-17T13:30:10.106477+0000","172.21.15.151:6800/4217080091":"2023-11-17T13:30:10.106477+0000","172.21.15.151:0/1874828182":"2023-11-17T13:30:10.106477+0000","172.21.15.151:6800/3823821624":"2023-11-17T13:29:52.404698+0000","172.21.15.151:6801/3823821624":"2023-11-17T13:29:52.404698+0000","172.21.15.151:0/1542985892":"2023-11-17T13:29:52.404698+0000","172.21.15.151:0/3861002685":"2023-11-17T13:29:52.404698+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-11-16T13:33:47.914 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:46.129177+0000 mgr.smithi151.fcujra (mgr.14164) 173 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:47.914 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:47.635746+0000 mon.smithi151 (mon.0) 591 : audit [DBG] from='client.? 172.21.15.151:0/1317700037' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-11-16T13:33:48.007 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-11-16T13:33:48.008 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-11-16T13:33:48.009 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-11-16T13:33:48.009 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-11-16T13:33:48.010 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-11-16T13:33:48.010 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-11-16T13:33:48.011 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-11-16T13:33:48.012 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-11-16T13:33:48.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:46.129177+0000 mgr.smithi151.fcujra (mgr.14164) 173 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:48.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:47.635746+0000 mon.smithi151 (mon.0) 591 : audit [DBG] from='client.? 172.21.15.151:0/1317700037' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-11-16T13:33:48.447 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:48.548 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:48.746 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:48.749 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:48.800 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:48.801 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:50.087 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:48.135433+0000 mgr.smithi151.fcujra (mgr.14164) 174 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:48.135433+0000 mgr.smithi151.fcujra (mgr.14164) 174 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:51.063 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:51.114 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:51.116 INFO:teuthology.orchestra.run.smithi151.stdout:193273528326 2023-11-16T13:33:51.117 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-11-16T13:33:51.964 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:51 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:50.135810+0000 mgr.smithi151.fcujra (mgr.14164) 175 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:51.997 INFO:teuthology.orchestra.run.smithi151.stdout:171798691848 2023-11-16T13:33:51.997 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-11-16T13:33:52.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:51 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:50.135810+0000 mgr.smithi151.fcujra (mgr.14164) 175 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:52.945 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:52.969 INFO:teuthology.orchestra.run.smithi151.stdout:60129542164 2023-11-16T13:33:52.970 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-11-16T13:33:53.264 INFO:teuthology.orchestra.run.smithi151.stdout:150323855371 2023-11-16T13:33:53.264 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-11-16T13:33:53.787 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:53.811 INFO:teuthology.orchestra.run.smithi151.stdout:111669149711 2023-11-16T13:33:53.812 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-11-16T13:33:53.999 INFO:teuthology.orchestra.run.smithi151.stdout:128849018893 2023-11-16T13:33:53.999 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-11-16T13:33:54.070 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:52.136125+0000 mgr.smithi151.fcujra (mgr.14164) 176 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:54.119 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:54.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:52.136125+0000 mgr.smithi151.fcujra (mgr.14164) 176 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:54.639 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:55.116 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:56.188 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:55 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:54.137050+0000 mgr.smithi151.fcujra (mgr.14164) 177 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:56.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:55 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:54.137050+0000 mgr.smithi151.fcujra (mgr.14164) 177 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:56.514 INFO:teuthology.orchestra.run.smithi151.stdout:81604378642 2023-11-16T13:33:56.514 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-11-16T13:33:56.736 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:56.830 INFO:teuthology.orchestra.run.smithi151.stdout:150323855371 2023-11-16T13:33:57.112 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:56 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:56.827654+0000 mon.smithi151 (mon.0) 592 : audit [DBG] from='client.? 172.21.15.151:0/3487890448' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-11-16T13:33:57.245 INFO:teuthology.orchestra.run.smithi151.stdout:171798691849 2023-11-16T13:33:57.304 INFO:teuthology.orchestra.run.smithi151.stdout:193273528327 2023-11-16T13:33:57.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:56 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:56.827654+0000 mon.smithi151 (mon.0) 592 : audit [DBG] from='client.? 172.21.15.151:0/3487890448' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-11-16T13:33:58.047 INFO:tasks.cephadm.ceph_manager.ceph:need seq 150323855371 got 150323855371 for osd.5 2023-11-16T13:33:58.047 DEBUG:teuthology.parallel:result is None 2023-11-16T13:33:58.162 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:56.137713+0000 mgr.smithi151.fcujra (mgr.14164) 178 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:58.163 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:57.244036+0000 mon.smithi151 (mon.0) 593 : audit [DBG] from='client.? 172.21.15.151:0/2254548602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-11-16T13:33:58.163 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:57.303682+0000 mon.smithi151 (mon.0) 594 : audit [DBG] from='client.? 172.21.15.151:0/3548219155' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-11-16T13:33:58.163 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:57.763575+0000 mon.smithi151 (mon.0) 595 : audit [DBG] from='client.? 172.21.15.151:0/1579873521' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-11-16T13:33:58.163 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:57.897722+0000 mon.smithi151 (mon.0) 596 : audit [DBG] from='client.? 172.21.15.151:0/2745797365' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-11-16T13:33:58.180 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691848 got 171798691849 for osd.6 2023-11-16T13:33:58.180 DEBUG:teuthology.parallel:result is None 2023-11-16T13:33:58.230 INFO:teuthology.orchestra.run.smithi151.stdout:38654705690 2023-11-16T13:33:58.230 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-11-16T13:33:58.266 INFO:teuthology.orchestra.run.smithi151.stdout:60129542165 2023-11-16T13:33:58.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:56.137713+0000 mgr.smithi151.fcujra (mgr.14164) 178 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:33:58.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:57.244036+0000 mon.smithi151 (mon.0) 593 : audit [DBG] from='client.? 172.21.15.151:0/2254548602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-11-16T13:33:58.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:57.303682+0000 mon.smithi151 (mon.0) 594 : audit [DBG] from='client.? 172.21.15.151:0/3548219155' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-11-16T13:33:58.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:57.763575+0000 mon.smithi151 (mon.0) 595 : audit [DBG] from='client.? 172.21.15.151:0/1579873521' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-11-16T13:33:58.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:57.897722+0000 mon.smithi151 (mon.0) 596 : audit [DBG] from='client.? 172.21.15.151:0/2745797365' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-11-16T13:33:58.685 INFO:tasks.cephadm.ceph_manager.ceph:need seq 193273528326 got 193273528327 for osd.7 2023-11-16T13:33:58.685 DEBUG:teuthology.parallel:result is None 2023-11-16T13:33:58.687 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542164 got 60129542165 for osd.1 2023-11-16T13:33:58.687 DEBUG:teuthology.parallel:result is None 2023-11-16T13:33:58.790 INFO:teuthology.orchestra.run.smithi151.stdout:128849018894 2023-11-16T13:33:59.179 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:59.211 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:33:59.302 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018893 got 128849018894 for osd.4 2023-11-16T13:33:59.391 DEBUG:teuthology.parallel:result is None 2023-11-16T13:33:59.551 INFO:teuthology.orchestra.run.smithi151.stdout:111669149712 2023-11-16T13:34:00.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:33:58.138168+0000 mgr.smithi151.fcujra (mgr.14164) 179 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:00.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:33:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:33:59.547112+0000 mon.smithi151 (mon.0) 597 : audit [DBG] from='client.? 172.21.15.151:0/3931399141' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-11-16T13:34:00.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:33:58.138168+0000 mgr.smithi151.fcujra (mgr.14164) 179 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:00.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:33:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:33:59.547112+0000 mon.smithi151 (mon.0) 597 : audit [DBG] from='client.? 172.21.15.151:0/3931399141' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-11-16T13:34:00.378 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149711 got 111669149712 for osd.3 2023-11-16T13:34:00.378 DEBUG:teuthology.parallel:result is None 2023-11-16T13:34:00.853 INFO:teuthology.orchestra.run.smithi151.stdout:81604378642 2023-11-16T13:34:00.962 INFO:teuthology.orchestra.run.smithi151.stdout:38654705691 2023-11-16T13:34:01.137 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:00.852510+0000 mon.smithi151 (mon.0) 598 : audit [DBG] from='client.? 172.21.15.151:0/4128735213' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-11-16T13:34:01.222 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378642 got 81604378642 for osd.2 2023-11-16T13:34:01.222 DEBUG:teuthology.parallel:result is None 2023-11-16T13:34:01.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:00.852510+0000 mon.smithi151 (mon.0) 598 : audit [DBG] from='client.? 172.21.15.151:0/4128735213' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-11-16T13:34:01.462 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705690 got 38654705691 for osd.0 2023-11-16T13:34:01.463 DEBUG:teuthology.parallel:result is None 2023-11-16T13:34:01.463 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-11-16T13:34:01.463 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph pg dump --format=json 2023-11-16T13:34:01.686 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:34:02.150 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:00.138454+0000 mgr.smithi151.fcujra (mgr.14164) 180 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:02.150 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:00.961694+0000 mon.smithi151 (mon.0) 599 : audit [DBG] from='client.? 172.21.15.151:0/3267728784' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-11-16T13:34:02.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:00.138454+0000 mgr.smithi151.fcujra (mgr.14164) 180 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:02.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:00.961694+0000 mon.smithi151 (mon.0) 599 : audit [DBG] from='client.? 172.21.15.151:0/3267728784' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-11-16T13:34:02.723 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:34:02.725 INFO:teuthology.orchestra.run.smithi151.stderr:dumped all 2023-11-16T13:34:03.086 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:02.139016+0000 mgr.smithi151.fcujra (mgr.14164) 181 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:03.086 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:02.720185+0000 mgr.smithi151.fcujra (mgr.14164) 182 : audit [DBG] from='client.14468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-11-16T13:34:03.088 INFO:teuthology.orchestra.run.smithi151.stdout:{"pg_ready":true,"pg_map":{"version":145,"stamp":"2023-11-16T13:34:02.138696+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":42712,"kb_used_data":3544,"kb_used_omap":0,"kb_used_meta":39104,"kb_avail":749918504,"statfs":{"total":767960285184,"available":767916548096,"internally_reserved":0,"allocated":3629056,"data_stored":1407578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40042496},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"1.0","version":"0'0","reported_seq":10,"reported_epoch":46,"state":"active+clean","last_fresh":"2023-11-16T13:33:37.219599+0000","last_change":"2023-11-16T13:33:37.219599+0000","last_active":"2023-11-16T13:33:37.219599+0000","last_peered":"2023-11-16T13:33:37.219599+0000","last_clean":"2023-11-16T13:33:37.219599+0000","last_became_active":"2023-11-16T13:33:37.219259+0000","last_became_peered":"2023-11-16T13:33:37.219259+0000","last_unstale":"2023-11-16T13:33:37.219599+0000","last_undegraded":"2023-11-16T13:33:37.219599+0000","last_fullsized":"2023-11-16T13:33:37.219599+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-11-16T13:32:42.943064+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-11-16T13:32:42.943064+0000","last_clean_scrub_stamp":"2023-11-16T13:32:42.943064+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":45,"seq":193273528329,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5188,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":4736,"kb_avail":93739964,"statfs":{"total":95995035648,"available":95989723136,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4849664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40100000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44400000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53400000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48199999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39100000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]}]},{"osd":6,"up_from":40,"seq":171798691850,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5252,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739900,"statfs":{"total":95995035648,"available":95989657600,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39100000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.496}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46500000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42999999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51200000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60699999999999998}]}]},{"osd":1,"up_from":14,"seq":60129542166,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5444,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739708,"statfs":{"total":95995035648,"available":95989460992,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Nov 16 13:33:18 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.29299999999999998}]},{"osd":2,"last update":"Thu Nov 16 13:33:38 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.372},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.27800000000000002}]},{"osd":3,"last update":"Thu Nov 16 13:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.8799999999999999,"5min":2.8799999999999999,"15min":2.8799999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.41699999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45000000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58899999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51100000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705691,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5508,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739644,"statfs":{"total":95995035648,"available":95989395456,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Nov 16 13:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.371}]},{"osd":2,"last update":"Thu Nov 16 13:33:38 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.41999999999999998}]},{"osd":3,"last update":"Thu Nov 16 13:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.4630000000000001,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":0.40999999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68500000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57999999999999996}]}]},{"osd":2,"up_from":19,"seq":81604378643,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5380,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739772,"statfs":{"total":95995035648,"available":95989526528,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Nov 16 13:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.375,"5min":0.375,"15min":0.375},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.27900000000000003}]},{"osd":1,"last update":"Thu Nov 16 13:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.376,"15min":0.376},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.29699999999999999}]},{"osd":3,"last update":"Thu Nov 16 13:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.315}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71299999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57799999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78600000000000003}]}]},{"osd":3,"up_from":26,"seq":111669149712,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5316,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739836,"statfs":{"total":95995035648,"available":95989592064,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Nov 16 13:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":3.621}]},{"osd":1,"last update":"Thu Nov 16 13:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":3.9769999999999999},{"interface":"front","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.38}]},{"osd":2,"last update":"Thu Nov 16 13:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.39900000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.9079999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.432}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.8690000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.988}]}]},{"osd":4,"up_from":30,"seq":128849018895,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5316,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739836,"statfs":{"total":95995035648,"available":95989592064,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39200000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40799999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55800000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57999999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51200000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57299999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.623}]}]},{"osd":5,"up_from":35,"seq":150323855373,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5308,"kb_used_data":436,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739844,"statfs":{"total":95995035648,"available":95989600256,"internally_reserved":0,"allocated":446464,"data_stored":175396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55600000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.625}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47599999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34999999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.436}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-11-16T13:34:03.091 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph pg dump --format=json 2023-11-16T13:34:03.314 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:34:03.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:02.139016+0000 mgr.smithi151.fcujra (mgr.14164) 181 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:03.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:02.720185+0000 mgr.smithi151.fcujra (mgr.14164) 182 : audit [DBG] from='client.14468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-11-16T13:34:04.412 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:34:04.414 INFO:teuthology.orchestra.run.smithi151.stderr:dumped all 2023-11-16T13:34:04.761 INFO:teuthology.orchestra.run.smithi151.stdout:{"pg_ready":true,"pg_map":{"version":146,"stamp":"2023-11-16T13:34:04.139189+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":42712,"kb_used_data":3544,"kb_used_omap":0,"kb_used_meta":39104,"kb_avail":749918504,"statfs":{"total":767960285184,"available":767916548096,"internally_reserved":0,"allocated":3629056,"data_stored":1407578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40042496},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"1.0","version":"0'0","reported_seq":10,"reported_epoch":46,"state":"active+clean","last_fresh":"2023-11-16T13:33:37.219599+0000","last_change":"2023-11-16T13:33:37.219599+0000","last_active":"2023-11-16T13:33:37.219599+0000","last_peered":"2023-11-16T13:33:37.219599+0000","last_clean":"2023-11-16T13:33:37.219599+0000","last_became_active":"2023-11-16T13:33:37.219259+0000","last_became_peered":"2023-11-16T13:33:37.219259+0000","last_unstale":"2023-11-16T13:33:37.219599+0000","last_undegraded":"2023-11-16T13:33:37.219599+0000","last_fullsized":"2023-11-16T13:33:37.219599+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-11-16T13:32:42.943064+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-11-16T13:32:42.943064+0000","last_clean_scrub_stamp":"2023-11-16T13:32:42.943064+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":45,"seq":193273528329,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5188,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":4736,"kb_avail":93739964,"statfs":{"total":95995035648,"available":95989723136,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4849664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40100000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44400000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53400000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48199999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39100000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]}]},{"osd":6,"up_from":40,"seq":171798691851,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5252,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739900,"statfs":{"total":95995035648,"available":95989657600,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.43099999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47399999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41299999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51500000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40100000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59499999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.502}]}]},{"osd":1,"up_from":14,"seq":60129542167,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5444,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739708,"statfs":{"total":95995035648,"available":95989460992,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Nov 16 13:33:18 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.48199999999999998}]},{"osd":2,"last update":"Thu Nov 16 13:33:38 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.44400000000000001}]},{"osd":3,"last update":"Thu Nov 16 13:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.8799999999999999,"5min":2.8799999999999999,"15min":2.8799999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.503}]},{"osd":4,"last update":"Thu Nov 16 13:34:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":0.70599999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.62}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63900000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.754}]}]},{"osd":0,"up_from":9,"seq":38654705692,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5508,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739644,"statfs":{"total":95995035648,"available":95989395456,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Nov 16 13:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.41399999999999998}]},{"osd":2,"last update":"Thu Nov 16 13:33:38 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.46400000000000002}]},{"osd":3,"last update":"Thu Nov 16 13:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.4630000000000001,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":0.33300000000000002}]},{"osd":4,"last update":"Thu Nov 16 13:33:59 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.57199999999999995}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54400000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.623}]}]},{"osd":2,"up_from":19,"seq":81604378643,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5380,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739772,"statfs":{"total":95995035648,"available":95989526528,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Nov 16 13:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.375,"5min":0.375,"15min":0.375},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.27900000000000003}]},{"osd":1,"last update":"Thu Nov 16 13:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.376,"15min":0.376},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.29699999999999999}]},{"osd":3,"last update":"Thu Nov 16 13:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.315}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71299999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57799999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78600000000000003}]}]},{"osd":3,"up_from":26,"seq":111669149713,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5316,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739836,"statfs":{"total":95995035648,"available":95989592064,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Nov 16 13:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.245},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.28599999999999998}]},{"osd":1,"last update":"Thu Nov 16 13:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.316},{"interface":"front","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Nov 16 13:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.29799999999999999}]},{"osd":4,"last update":"Thu Nov 16 13:34:00 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":2.9079999999999999,"5min":2.9079999999999999,"15min":2.9079999999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":3.9060000000000001,"5min":3.9060000000000001,"15min":3.9060000000000001},"last":0.42499999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36799999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39100000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018896,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5316,"kb_used_data":444,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739836,"statfs":{"total":95995035648,"available":95989592064,"internally_reserved":0,"allocated":454656,"data_stored":176026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Nov 16 13:34:00 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.42199999999999999}]},{"osd":1,"last update":"Thu Nov 16 13:34:00 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.36599999999999999}]},{"osd":2,"last update":"Thu Nov 16 13:34:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.47299999999999998}]},{"osd":3,"last update":"Thu Nov 16 13:34:00 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.377},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.45600000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.438}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.505}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54800000000000004}]}]},{"osd":5,"up_from":35,"seq":150323855373,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5308,"kb_used_data":436,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739844,"statfs":{"total":95995035648,"available":95989600256,"internally_reserved":0,"allocated":446464,"data_stored":175396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55600000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.625}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47599999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34999999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.436}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-11-16T13:34:04.762 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-11-16T13:34:04.762 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-11-16T13:34:04.762 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-11-16T13:34:04.762 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph health --format=json 2023-11-16T13:34:04.983 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/mon.smithi151/config 2023-11-16T13:34:05.400 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:04.139417+0000 mgr.smithi151.fcujra (mgr.14164) 183 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:05.400 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:04.409597+0000 mgr.smithi151.fcujra (mgr.14164) 184 : audit [DBG] from='client.14472 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-11-16T13:34:05.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:04.139417+0000 mgr.smithi151.fcujra (mgr.14164) 183 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:05.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:04.409597+0000 mgr.smithi151.fcujra (mgr.14164) 184 : audit [DBG] from='client.14472 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-11-16T13:34:06.168 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:34:06.528 INFO:teuthology.orchestra.run.smithi151.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-11-16T13:34:06.528 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-11-16T13:34:06.529 INFO:tasks.cephadm:Setup complete, yielding 2023-11-16T13:34:06.529 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-11-16T13:34:06.574 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi151.front.sepia.ceph.com 2023-11-16T13:34:06.600 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- bash -c 'ceph orch status' 2023-11-16T13:34:07.216 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:06 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:06.167357+0000 mon.smithi151 (mon.0) 600 : audit [DBG] from='client.? 172.21.15.151:0/3893066681' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-11-16T13:34:07.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:06.167357+0000 mon.smithi151 (mon.0) 600 : audit [DBG] from='client.? 172.21.15.151:0/3893066681' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-11-16T13:34:07.911 INFO:teuthology.orchestra.run.smithi151.stdout:Backend: cephadm 2023-11-16T13:34:07.911 INFO:teuthology.orchestra.run.smithi151.stdout:Available: Yes 2023-11-16T13:34:07.911 INFO:teuthology.orchestra.run.smithi151.stdout:Paused: No 2023-11-16T13:34:08.177 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:06.139974+0000 mgr.smithi151.fcujra (mgr.14164) 185 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:08.278 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- bash -c 'ceph orch ps' 2023-11-16T13:34:08.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:06.139974+0000 mgr.smithi151.fcujra (mgr.14164) 185 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:09.157 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:08 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:07.909380+0000 mgr.smithi151.fcujra (mgr.14164) 186 : audit [DBG] from='client.14480 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:09.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:07.909380+0000 mgr.smithi151.fcujra (mgr.14164) 186 : audit [DBG] from='client.14480 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:09.537 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:34:15.735 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (2m) 56s ago 3m 12.2M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:34:15.735 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (3m) 56s ago 3m 7008k - 16.2.5 6933c2a0b7dd 954bd32d4ee2 2023-11-16T13:34:15.735 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (2m) 37s ago 2m 7155k - 16.2.5 6933c2a0b7dd 649801b49aea 2023-11-16T13:34:15.736 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (2m) 56s ago 3m 29.0M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:34:15.736 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:9283 running (4m) 56s ago 4m 432M - 16.2.5 6933c2a0b7dd a009db54a97f 2023-11-16T13:34:15.736 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (2m) 37s ago 2m 375M - 16.2.5 6933c2a0b7dd a6cdf8e550f9 2023-11-16T13:34:15.736 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (4m) 56s ago 4m 74.0M 2048M 16.2.5 6933c2a0b7dd 3a504a936c5d 2023-11-16T13:34:15.736 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (2m) 37s ago 2m 73.0M 2048M 16.2.5 6933c2a0b7dd 448b57d6232a 2023-11-16T13:34:15.736 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (2m) 56s ago 2m 18.1M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:34:15.737 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (2m) 37s ago 2m 18.2M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:34:15.737 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (2m) 56s ago 2m 35.4M 3224M 16.2.5 6933c2a0b7dd 7598578fb701 2023-11-16T13:34:15.737 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (117s) 56s ago 117s 34.8M 3224M 16.2.5 6933c2a0b7dd 989f26649fd3 2023-11-16T13:34:15.737 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (101s) 56s ago 101s 34.3M 3224M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:34:15.737 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (87s) 56s ago 87s 31.7M 3224M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:34:15.737 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (76s) 37s ago 75s 32.8M 3992M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:34:15.738 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (64s) 37s ago 63s 31.4M 3992M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:34:15.738 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (51s) 37s ago 51s 30.0M 3992M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:34:15.738 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (39s) 37s ago 38s 14.5M 3992M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:34:15.738 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (2m) 56s ago 2m 34.1M - 2.18.1 de242295e225 336d5d209623 2023-11-16T13:34:15.740 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:08.140355+0000 mgr.smithi151.fcujra (mgr.14164) 187 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:15.741 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:09.529071+0000 mgr.smithi151.fcujra (mgr.14164) 188 : audit [DBG] from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:15.741 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:10.140866+0000 mgr.smithi151.fcujra (mgr.14164) 189 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:15.741 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:12.141373+0000 mgr.smithi151.fcujra (mgr.14164) 190 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:15.743 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- bash -c 'ceph orch ls' 2023-11-16T13:34:15.780 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:08.140355+0000 mgr.smithi151.fcujra (mgr.14164) 187 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:15.780 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:09.529071+0000 mgr.smithi151.fcujra (mgr.14164) 188 : audit [DBG] from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:15.780 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:10.140866+0000 mgr.smithi151.fcujra (mgr.14164) 189 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:15.780 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:12.141373+0000 mgr.smithi151.fcujra (mgr.14164) 190 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:16.105 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:14.141942+0000 mgr.smithi151.fcujra (mgr.14164) 191 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:16.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:14.141942+0000 mgr.smithi151.fcujra (mgr.14164) 191 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:17.137 INFO:teuthology.orchestra.run.smithi151.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2023-11-16T13:34:17.138 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager ?:9093,9094 1/1 64s ago 4m count:1 2023-11-16T13:34:17.138 INFO:teuthology.orchestra.run.smithi151.stdout:crash 2/2 64s ago 4m * 2023-11-16T13:34:17.138 INFO:teuthology.orchestra.run.smithi151.stdout:grafana ?:3000 1/1 64s ago 4m count:1 2023-11-16T13:34:17.138 INFO:teuthology.orchestra.run.smithi151.stdout:mgr 2/2 64s ago 4m count:2 2023-11-16T13:34:17.138 INFO:teuthology.orchestra.run.smithi151.stdout:mon 2/2 64s ago 3m smithi151:172.21.15.151=smithi151;smithi167:172.21.15.167=smithi167;count:2 2023-11-16T13:34:17.138 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter ?:9100 2/2 64s ago 4m * 2023-11-16T13:34:17.138 INFO:teuthology.orchestra.run.smithi151.stdout:osd 8/8 64s ago - 2023-11-16T13:34:17.138 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus ?:9095 1/1 64s ago 4m count:1 2023-11-16T13:34:17.157 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:16.142262+0000 mgr.smithi151.fcujra (mgr.14164) 192 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:17.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:16.142262+0000 mgr.smithi151.fcujra (mgr.14164) 192 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:17.511 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- bash -c 'ceph orch host ls' 2023-11-16T13:34:18.148 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:17.130741+0000 mgr.smithi151.fcujra (mgr.14164) 193 : audit [DBG] from='client.14488 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:18.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:17.130741+0000 mgr.smithi151.fcujra (mgr.14164) 193 : audit [DBG] from='client.14488 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:18.819 INFO:teuthology.orchestra.run.smithi151.stdout:HOST ADDR LABELS STATUS 2023-11-16T13:34:18.819 INFO:teuthology.orchestra.run.smithi151.stdout:smithi151 172.21.15.151 2023-11-16T13:34:18.819 INFO:teuthology.orchestra.run.smithi151.stdout:smithi167 172.21.15.167 2023-11-16T13:34:19.186 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:18.142605+0000 mgr.smithi151.fcujra (mgr.14164) 194 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:19.186 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:18.817157+0000 mgr.smithi151.fcujra (mgr.14164) 195 : audit [DBG] from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:19.187 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- bash -c 'ceph orch device ls' 2023-11-16T13:34:19.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:18.142605+0000 mgr.smithi151.fcujra (mgr.14164) 194 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:19.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:18.817157+0000 mgr.smithi151.fcujra (mgr.14164) 195 : audit [DBG] from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:20.534 INFO:teuthology.orchestra.run.smithi151.stdout:Hostname Path Type Serial Size Health Ident Fault Available 2023-11-16T13:34:20.534 INFO:teuthology.orchestra.run.smithi151.stdout:smithi151 /dev/nvme0n1 ssd CVFT623300JJ400BGN 400G Unknown N/A N/A No 2023-11-16T13:34:20.534 INFO:teuthology.orchestra.run.smithi151.stdout:smithi167 /dev/nvme0n1 ssd CVFT62330080400BGN 400G Unknown N/A N/A No 2023-11-16T13:34:21.054 INFO:teuthology.run_tasks:Running task vip.exec... 2023-11-16T13:34:21.125 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi151.front.sepia.ceph.com 2023-11-16T13:34:21.126 DEBUG:teuthology.orchestra.run.smithi151:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2023-11-16T13:34:21.156 INFO:teuthology.orchestra.run.smithi151.stderr:+ systemctl stop nfs-server 2023-11-16T13:34:21.163 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi167.front.sepia.ceph.com 2023-11-16T13:34:21.163 DEBUG:teuthology.orchestra.run.smithi167:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2023-11-16T13:34:21.202 INFO:teuthology.orchestra.run.smithi167.stderr:+ systemctl stop nfs-server 2023-11-16T13:34:21.210 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-11-16T13:34:21.221 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi151.front.sepia.ceph.com 2023-11-16T13:34:21.221 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- bash -c 'ceph fs volume create foofs' 2023-11-16T13:34:22.139 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:20.143081+0000 mgr.smithi151.fcujra (mgr.14164) 196 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:22.140 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:20.531944+0000 mgr.smithi151.fcujra (mgr.14164) 197 : audit [DBG] from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:22.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:20.143081+0000 mgr.smithi151.fcujra (mgr.14164) 196 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:22.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:20.531944+0000 mgr.smithi151.fcujra (mgr.14164) 197 : audit [DBG] from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:22.247199+0000 mon.smithi151 (mon.0) 601 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:34:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:22.247493+0000 mon.smithi151 (mon.0) 602 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:34:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:22.503191+0000 mon.smithi151 (mon.0) 603 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2023-11-16T13:34:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:22.247199+0000 mon.smithi151 (mon.0) 601 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:34:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:22.247493+0000 mon.smithi151 (mon.0) 602 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:34:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:22.503191+0000 mon.smithi151 (mon.0) 603 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2023-11-16T13:34:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:34:23.917+0000 7f835215b700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-11-16T13:34:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:22.143443+0000 mgr.smithi151.fcujra (mgr.14164) 198 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:22.502651+0000 mgr.smithi151.fcujra (mgr.14164) 199 : audit [DBG] from='client.14500 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:22.904316+0000 mon.smithi151 (mon.0) 604 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2023-11-16T13:34:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:22.904353+0000 mon.smithi151 (mon.0) 605 : cluster [DBG] osdmap e47: 8 total, 8 up, 8 in 2023-11-16T13:34:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:22.906550+0000 mon.smithi151 (mon.0) 606 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2023-11-16T13:34:24.323 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2023-11-16T13:34:24.332 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2023-11-16T13:34:24.332 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch ls -f json 2023-11-16T13:34:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:22.143443+0000 mgr.smithi151.fcujra (mgr.14164) 198 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:22.502651+0000 mgr.smithi151.fcujra (mgr.14164) 199 : audit [DBG] from='client.14500 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:22.904316+0000 mon.smithi151 (mon.0) 604 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2023-11-16T13:34:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:22.904353+0000 mon.smithi151 (mon.0) 605 : cluster [DBG] osdmap e47: 8 total, 8 up, 8 in 2023-11-16T13:34:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:22.906550+0000 mon.smithi151 (mon.0) 606 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2023-11-16T13:34:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:23.907309+0000 mon.smithi151 (mon.0) 607 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2023-11-16T13:34:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:23.907526+0000 mon.smithi151 (mon.0) 608 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-11-16T13:34:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:23.917297+0000 mon.smithi151 (mon.0) 609 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2023-11-16T13:34:25.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:23.918186+0000 mon.smithi151 (mon.0) 610 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-11-16T13:34:25.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:23.918221+0000 mon.smithi151 (mon.0) 611 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-11-16T13:34:25.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:23.934926+0000 mon.smithi151 (mon.0) 612 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-11-16T13:34:25.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:23.935094+0000 mon.smithi151 (mon.0) 613 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2023-11-16T13:34:25.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:23.935180+0000 mon.smithi151 (mon.0) 614 : cluster [DBG] fsmap foofs:0 2023-11-16T13:34:25.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:23.939987+0000 mgr.smithi151.fcujra (mgr.14164) 200 : cephadm [INF] Saving service mds.foofs spec with placement count:2 2023-11-16T13:34:25.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:23.945861+0000 mon.smithi151 (mon.0) 615 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:25.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:23.946616+0000 mon.smithi151 (mon.0) 616 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:25.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:23.949231+0000 mon.smithi151 (mon.0) 617 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:25.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:23.950487+0000 mon.smithi151 (mon.0) 618 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:25.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:24.143766+0000 mgr.smithi151.fcujra (mgr.14164) 201 : cluster [DBG] pgmap v159: 65 pgs: 2 creating+activating, 13 creating+peering, 4 active+clean, 46 unknown; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:25.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:24.318622+0000 mon.smithi151 (mon.0) 619 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:25.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:24.325795+0000 mon.smithi151 (mon.0) 620 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:25.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:24.326437+0000 mon.smithi151 (mon.0) 621 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi151.mdkcfw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-11-16T13:34:25.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:24.329369+0000 mon.smithi151 (mon.0) 622 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi151.mdkcfw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-11-16T13:34:25.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:24.329902+0000 mon.smithi151 (mon.0) 623 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:25.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:24.330442+0000 mgr.smithi151.fcujra (mgr.14164) 202 : cephadm [INF] Deploying daemon mds.foofs.smithi151.mdkcfw on smithi151 2023-11-16T13:34:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:23.907309+0000 mon.smithi151 (mon.0) 607 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2023-11-16T13:34:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:23.907526+0000 mon.smithi151 (mon.0) 608 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-11-16T13:34:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:23.917297+0000 mon.smithi151 (mon.0) 609 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2023-11-16T13:34:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:23.918186+0000 mon.smithi151 (mon.0) 610 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-11-16T13:34:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:23.918221+0000 mon.smithi151 (mon.0) 611 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-11-16T13:34:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:23.934926+0000 mon.smithi151 (mon.0) 612 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-11-16T13:34:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:23.935094+0000 mon.smithi151 (mon.0) 613 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2023-11-16T13:34:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:23.935180+0000 mon.smithi151 (mon.0) 614 : cluster [DBG] fsmap foofs:0 2023-11-16T13:34:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:23.939987+0000 mgr.smithi151.fcujra (mgr.14164) 200 : cephadm [INF] Saving service mds.foofs spec with placement count:2 2023-11-16T13:34:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:23.945861+0000 mon.smithi151 (mon.0) 615 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:23.946616+0000 mon.smithi151 (mon.0) 616 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:23.949231+0000 mon.smithi151 (mon.0) 617 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:23.950487+0000 mon.smithi151 (mon.0) 618 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:24.143766+0000 mgr.smithi151.fcujra (mgr.14164) 201 : cluster [DBG] pgmap v159: 65 pgs: 2 creating+activating, 13 creating+peering, 4 active+clean, 46 unknown; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:25.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:24.318622+0000 mon.smithi151 (mon.0) 619 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:25.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:24.325795+0000 mon.smithi151 (mon.0) 620 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:25.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:24.326437+0000 mon.smithi151 (mon.0) 621 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi151.mdkcfw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-11-16T13:34:25.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:24.329369+0000 mon.smithi151 (mon.0) 622 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi151.mdkcfw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-11-16T13:34:25.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:24.329902+0000 mon.smithi151 (mon.0) 623 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:25.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:24.330442+0000 mgr.smithi151.fcujra (mgr.14164) 202 : cephadm [INF] Deploying daemon mds.foofs.smithi151.mdkcfw on smithi151 2023-11-16T13:34:26.025 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:34:26.025 INFO:teuthology.orchestra.run.smithi151.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2023-11-16T13:30:13.774692Z", "last_refresh": "2023-11-16T13:33:12.593040Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2023-11-16T13:30:03.539779Z", "last_refresh": "2023-11-16T13:33:12.593099Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2023-11-16T13:30:12.037972Z", "last_refresh": "2023-11-16T13:33:12.593170Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2023-11-16T13:34:23.946032Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2023-11-16T13:34:23.940012Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2023-11-16T13:30:02.671949Z", "last_refresh": "2023-11-16T13:33:12.592974Z", "running": 2, "size": 2}}, {"events": ["2023-11-16T13:30:38.528338Z service:mon [INFO] \"service was created\""], "placement": {"count": 2, "hosts": ["smithi151:172.21.15.151=smithi151", "smithi167:172.21.15.167=smithi167"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2023-11-16T13:30:38.525496Z", "last_refresh": "2023-11-16T13:33:12.592800Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2023-11-16T13:30:12.907869Z", "last_refresh": "2023-11-16T13:33:12.593224Z", "ports": [9100], "running": 2, "size": 2}}, {"placement": {}, "service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "6933c2a0b7ddc222e77dc8d9dc471a0c639a1c0bded5077be53ad3a9557b4355", "container_image_name": "docker.io/ceph/ceph@sha256:829ebf54704f2d827de00913b171e5da741aad9b53c1f35ad59251524790eceb", "last_refresh": "2023-11-16T13:33:12.593331Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2023-11-16T13:30:11.164903Z", "last_refresh": "2023-11-16T13:33:12.593277Z", "ports": [9095], "running": 1, "size": 1}}] 2023-11-16T13:34:26.230 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:25 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:24.935358+0000 mon.smithi151 (mon.0) 624 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-11-16T13:34:26.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:25 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:24.935358+0000 mon.smithi151 (mon.0) 624 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-11-16T13:34:26.468 INFO:tasks.cephadm:mds.foofs has 0/2 2023-11-16T13:34:27.151 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:25.940187+0000 mon.smithi151 (mon.0) 625 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-11-16T13:34:27.151 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:26.020135+0000 mgr.smithi151.fcujra (mgr.14164) 203 : audit [DBG] from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-11-16T13:34:27.151 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:26.144069+0000 mgr.smithi151.fcujra (mgr.14164) 204 : cluster [DBG] pgmap v162: 65 pgs: 2 creating+activating, 13 creating+peering, 8 active+clean, 42 unknown; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:27.152 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:26.159373+0000 mon.smithi151 (mon.0) 626 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:27.152 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:26.160338+0000 mon.smithi151 (mon.0) 627 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi167.irznoa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-11-16T13:34:27.152 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:26.165206+0000 mon.smithi151 (mon.0) 628 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi167.irznoa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-11-16T13:34:27.152 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:26.165789+0000 mon.smithi151 (mon.0) 629 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:27.152 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:26.166303+0000 mgr.smithi151.fcujra (mgr.14164) 205 : cephadm [INF] Deploying daemon mds.foofs.smithi167.irznoa on smithi167 2023-11-16T13:34:27.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:25.940187+0000 mon.smithi151 (mon.0) 625 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-11-16T13:34:27.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:26.020135+0000 mgr.smithi151.fcujra (mgr.14164) 203 : audit [DBG] from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-11-16T13:34:27.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:26.144069+0000 mgr.smithi151.fcujra (mgr.14164) 204 : cluster [DBG] pgmap v162: 65 pgs: 2 creating+activating, 13 creating+peering, 8 active+clean, 42 unknown; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:27.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:26.159373+0000 mon.smithi151 (mon.0) 626 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:27.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:26.160338+0000 mon.smithi151 (mon.0) 627 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi167.irznoa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-11-16T13:34:27.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:26.165206+0000 mon.smithi151 (mon.0) 628 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi167.irznoa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-11-16T13:34:27.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:26.165789+0000 mon.smithi151 (mon.0) 629 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:27.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:26.166303+0000 mgr.smithi151.fcujra (mgr.14164) 205 : cephadm [INF] Deploying daemon mds.foofs.smithi167.irznoa on smithi167 2023-11-16T13:34:27.469 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch ls -f json 2023-11-16T13:34:28.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:27.702131+0000 mon.smithi151 (mon.0) 630 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:28.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:27.703134+0000 mon.smithi151 (mon.0) 631 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:28.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:27.703994+0000 mon.smithi151 (mon.0) 632 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:28.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:27.704546+0000 mon.smithi151 (mon.0) 633 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:28.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:27.948008+0000 mon.smithi151 (mon.0) 634 : cluster [DBG] mds.? [v2:172.21.15.167:6832/2149060410,v1:172.21.15.167:6833/2149060410] up:boot 2023-11-16T13:34:28.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:27.948066+0000 mon.smithi151 (mon.0) 635 : cluster [INF] daemon mds.foofs.smithi167.irznoa assigned to filesystem foofs as rank 0 (now has 1 ranks) 2023-11-16T13:34:28.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:27.948130+0000 mon.smithi151 (mon.0) 636 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-11-16T13:34:28.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:27.948146+0000 mon.smithi151 (mon.0) 637 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-11-16T13:34:28.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:27.948156+0000 mon.smithi151 (mon.0) 638 : cluster [INF] Cluster is now healthy 2023-11-16T13:34:28.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:27.948327+0000 mon.smithi151 (mon.0) 639 : cluster [DBG] fsmap foofs:0 1 up:standby 2023-11-16T13:34:28.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:27.948412+0000 mon.smithi151 (mon.0) 640 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:34:28.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:27.950950+0000 mon.smithi151 (mon.0) 641 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi167.irznoa=up:creating} 2023-11-16T13:34:28.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:27.960875+0000 mon.smithi151 (mon.0) 642 : cluster [INF] daemon mds.foofs.smithi167.irznoa is now active in filesystem foofs as rank 0 2023-11-16T13:34:28.871 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:34:28.871 INFO:teuthology.orchestra.run.smithi151.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2023-11-16T13:30:13.774692Z", "last_refresh": "2023-11-16T13:33:12.593040Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2023-11-16T13:30:03.539779Z", "last_refresh": "2023-11-16T13:33:12.593099Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2023-11-16T13:30:12.037972Z", "last_refresh": "2023-11-16T13:33:12.593170Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2023-11-16T13:34:23.946032Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2023-11-16T13:34:23.940012Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2023-11-16T13:30:02.671949Z", "last_refresh": "2023-11-16T13:33:12.592974Z", "running": 2, "size": 2}}, {"events": ["2023-11-16T13:30:38.528338Z service:mon [INFO] \"service was created\""], "placement": {"count": 2, "hosts": ["smithi151:172.21.15.151=smithi151", "smithi167:172.21.15.167=smithi167"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2023-11-16T13:30:38.525496Z", "last_refresh": "2023-11-16T13:33:12.592800Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2023-11-16T13:30:12.907869Z", "last_refresh": "2023-11-16T13:33:12.593224Z", "ports": [9100], "running": 2, "size": 2}}, {"placement": {}, "service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "6933c2a0b7ddc222e77dc8d9dc471a0c639a1c0bded5077be53ad3a9557b4355", "container_image_name": "docker.io/ceph/ceph@sha256:829ebf54704f2d827de00913b171e5da741aad9b53c1f35ad59251524790eceb", "last_refresh": "2023-11-16T13:33:12.593331Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2023-11-16T13:30:11.164903Z", "last_refresh": "2023-11-16T13:33:12.593277Z", "ports": [9095], "running": 1, "size": 1}}] 2023-11-16T13:34:28.894 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:27.702131+0000 mon.smithi151 (mon.0) 630 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:28.895 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:27.703134+0000 mon.smithi151 (mon.0) 631 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:28.895 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:27.703994+0000 mon.smithi151 (mon.0) 632 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:28.895 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:27.704546+0000 mon.smithi151 (mon.0) 633 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:28.895 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:27.948008+0000 mon.smithi151 (mon.0) 634 : cluster [DBG] mds.? [v2:172.21.15.167:6832/2149060410,v1:172.21.15.167:6833/2149060410] up:boot 2023-11-16T13:34:28.895 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:27.948066+0000 mon.smithi151 (mon.0) 635 : cluster [INF] daemon mds.foofs.smithi167.irznoa assigned to filesystem foofs as rank 0 (now has 1 ranks) 2023-11-16T13:34:28.895 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:27.948130+0000 mon.smithi151 (mon.0) 636 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-11-16T13:34:28.895 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:27.948146+0000 mon.smithi151 (mon.0) 637 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-11-16T13:34:28.895 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:27.948156+0000 mon.smithi151 (mon.0) 638 : cluster [INF] Cluster is now healthy 2023-11-16T13:34:28.895 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:27.948327+0000 mon.smithi151 (mon.0) 639 : cluster [DBG] fsmap foofs:0 1 up:standby 2023-11-16T13:34:28.895 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:27.948412+0000 mon.smithi151 (mon.0) 640 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:34:28.896 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:27.950950+0000 mon.smithi151 (mon.0) 641 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi167.irznoa=up:creating} 2023-11-16T13:34:28.896 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:27.960875+0000 mon.smithi151 (mon.0) 642 : cluster [INF] daemon mds.foofs.smithi167.irznoa is now active in filesystem foofs as rank 0 2023-11-16T13:34:29.358 INFO:tasks.cephadm:mds.foofs has 2/2 2023-11-16T13:34:29.358 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-11-16T13:34:29.368 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi151.front.sepia.ceph.com 2023-11-16T13:34:29.369 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2023-11-16T13:34:30.089 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:28.144434+0000 mgr.smithi151.fcujra (mgr.14164) 206 : cluster [DBG] pgmap v163: 65 pgs: 2 creating+activating, 13 creating+peering, 39 active+clean, 11 unknown; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:30.089 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:28.866420+0000 mgr.smithi151.fcujra (mgr.14164) 207 : audit [DBG] from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-11-16T13:34:30.090 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:28.880880+0000 mon.smithi151 (mon.0) 643 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:30.090 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:28.957325+0000 mon.smithi151 (mon.0) 644 : cluster [DBG] mds.? [v2:172.21.15.167:6832/2149060410,v1:172.21.15.167:6833/2149060410] up:active 2023-11-16T13:34:30.090 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:28.957364+0000 mon.smithi151 (mon.0) 645 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 2023-11-16T13:34:30.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:28.144434+0000 mgr.smithi151.fcujra (mgr.14164) 206 : cluster [DBG] pgmap v163: 65 pgs: 2 creating+activating, 13 creating+peering, 39 active+clean, 11 unknown; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:30.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:28.866420+0000 mgr.smithi151.fcujra (mgr.14164) 207 : audit [DBG] from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-11-16T13:34:30.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:28.880880+0000 mon.smithi151 (mon.0) 643 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:30.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:28.957325+0000 mon.smithi151 (mon.0) 644 : cluster [DBG] mds.? [v2:172.21.15.167:6832/2149060410,v1:172.21.15.167:6833/2149060410] up:active 2023-11-16T13:34:30.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:28.957364+0000 mon.smithi151 (mon.0) 645 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 2023-11-16T13:34:32.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:30.144773+0000 mgr.smithi151.fcujra (mgr.14164) 208 : cluster [DBG] pgmap v164: 65 pgs: 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:32.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:30.883452+0000 mon.smithi151 (mon.0) 646 : cluster [DBG] mds.? [v2:172.21.15.151:6834/3865825489,v1:172.21.15.151:6835/3865825489] up:boot 2023-11-16T13:34:32.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:30.883712+0000 mon.smithi151 (mon.0) 647 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 1 up:standby 2023-11-16T13:34:32.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:30.883794+0000 mon.smithi151 (mon.0) 648 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi151.mdkcfw"}]: dispatch 2023-11-16T13:34:32.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:30.886342+0000 mon.smithi151 (mon.0) 649 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 1 up:standby 2023-11-16T13:34:32.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:30.993480+0000 mon.smithi151 (mon.0) 650 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pool create", "pool": "nfs-ganesha"}]: dispatch 2023-11-16T13:34:32.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:31.243208+0000 mon.smithi151 (mon.0) 651 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:32.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:31.246434+0000 mon.smithi151 (mon.0) 652 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:32.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:31.249763+0000 mon.smithi151 (mon.0) 653 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:32.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:31.250681+0000 mon.smithi151 (mon.0) 654 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:32.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:31.251237+0000 mon.smithi151 (mon.0) 655 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:32.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:31.254820+0000 mon.smithi151 (mon.0) 656 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:32.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:30.144773+0000 mgr.smithi151.fcujra (mgr.14164) 208 : cluster [DBG] pgmap v164: 65 pgs: 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2023-11-16T13:34:32.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:30.883452+0000 mon.smithi151 (mon.0) 646 : cluster [DBG] mds.? [v2:172.21.15.151:6834/3865825489,v1:172.21.15.151:6835/3865825489] up:boot 2023-11-16T13:34:32.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:30.883712+0000 mon.smithi151 (mon.0) 647 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 1 up:standby 2023-11-16T13:34:32.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:30.883794+0000 mon.smithi151 (mon.0) 648 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi151.mdkcfw"}]: dispatch 2023-11-16T13:34:32.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:30.886342+0000 mon.smithi151 (mon.0) 649 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 1 up:standby 2023-11-16T13:34:32.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:30.993480+0000 mon.smithi151 (mon.0) 650 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pool create", "pool": "nfs-ganesha"}]: dispatch 2023-11-16T13:34:32.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:31.243208+0000 mon.smithi151 (mon.0) 651 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:32.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:31.246434+0000 mon.smithi151 (mon.0) 652 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:32.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:31.249763+0000 mon.smithi151 (mon.0) 653 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:32.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:31.250681+0000 mon.smithi151 (mon.0) 654 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:32.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:31.251237+0000 mon.smithi151 (mon.0) 655 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:32.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:31.254820+0000 mon.smithi151 (mon.0) 656 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:32.934 INFO:teuthology.orchestra.run.smithi151.stdout:NFS Cluster Created Successfully 2023-11-16T13:34:33.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:30.992237+0000 mgr.smithi151.fcujra (mgr.14164) 209 : audit [DBG] from='client.14518 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "clusterid": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:33.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:31.900943+0000 mon.smithi151 (mon.0) 657 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pool create", "pool": "nfs-ganesha"}]': finished 2023-11-16T13:34:33.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:31.900995+0000 mon.smithi151 (mon.0) 658 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-11-16T13:34:33.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:31.902043+0000 mon.smithi151 (mon.0) 659 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pool application enable", "pool": "nfs-ganesha", "app": "nfs"}]: dispatch 2023-11-16T13:34:33.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:32.262369+0000 mon.smithi151 (mon.0) 660 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:33.322 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2023-11-16T13:34:33.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:30.992237+0000 mgr.smithi151.fcujra (mgr.14164) 209 : audit [DBG] from='client.14518 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "clusterid": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:33.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:31.900943+0000 mon.smithi151 (mon.0) 657 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pool create", "pool": "nfs-ganesha"}]': finished 2023-11-16T13:34:33.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:31.900995+0000 mon.smithi151 (mon.0) 658 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-11-16T13:34:33.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:31.902043+0000 mon.smithi151 (mon.0) 659 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pool application enable", "pool": "nfs-ganesha", "app": "nfs"}]: dispatch 2023-11-16T13:34:33.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:32.262369+0000 mon.smithi151 (mon.0) 660 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:34.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:32.145311+0000 mgr.smithi151.fcujra (mgr.14164) 210 : cluster [DBG] pgmap v166: 97 pgs: 32 unknown, 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 3 op/s 2023-11-16T13:34:34.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:32.904378+0000 mon.smithi151 (mon.0) 661 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pool application enable", "pool": "nfs-ganesha", "app": "nfs"}]': finished 2023-11-16T13:34:34.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:32.904554+0000 mon.smithi151 (mon.0) 662 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-11-16T13:34:34.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:32.933303+0000 mon.smithi151 (mon.0) 663 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:34.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:32.933911+0000 mon.smithi151 (mon.0) 664 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:34.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:32.935470+0000 mon.smithi151 (mon.0) 665 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:34.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:32.938320+0000 mon.smithi151 (mon.0) 666 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:34.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:32.949726+0000 mon.smithi151 (mon.0) 667 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:34.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:32.956588+0000 mon.smithi151 (mon.0) 668 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:34.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:32.977058+0000 mon.smithi151 (mon.0) 669 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi151.cywuiq", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]: dispatch 2023-11-16T13:34:34.166 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:32.981037+0000 mon.smithi151 (mon.0) 670 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi151.cywuiq", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]': finished 2023-11-16T13:34:34.166 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:32.981894+0000 mon.smithi151 (mon.0) 671 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]: dispatch 2023-11-16T13:34:34.166 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:32.988062+0000 mon.smithi151 (mon.0) 672 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]': finished 2023-11-16T13:34:34.166 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:32.989246+0000 mon.smithi151 (mon.0) 673 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:34.166 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:33.034074+0000 mon.smithi151 (mon.0) 674 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2023-11-16T13:34:34.166 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:33.037412+0000 mon.smithi151 (mon.0) 675 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2023-11-16T13:34:34.166 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:33.080238+0000 mon.smithi151 (mon.0) 676 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi151.cywuiq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2023-11-16T13:34:34.166 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:33.083322+0000 mon.smithi151 (mon.0) 677 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi151.cywuiq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2023-11-16T13:34:34.166 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:33.087148+0000 mon.smithi151 (mon.0) 678 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:34.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:32.145311+0000 mgr.smithi151.fcujra (mgr.14164) 210 : cluster [DBG] pgmap v166: 97 pgs: 32 unknown, 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 3 op/s 2023-11-16T13:34:34.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:32.904378+0000 mon.smithi151 (mon.0) 661 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pool application enable", "pool": "nfs-ganesha", "app": "nfs"}]': finished 2023-11-16T13:34:34.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:32.904554+0000 mon.smithi151 (mon.0) 662 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-11-16T13:34:34.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:32.933303+0000 mon.smithi151 (mon.0) 663 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:34.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:32.933911+0000 mon.smithi151 (mon.0) 664 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:34.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:32.935470+0000 mon.smithi151 (mon.0) 665 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:34.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:32.938320+0000 mon.smithi151 (mon.0) 666 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:34.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:32.949726+0000 mon.smithi151 (mon.0) 667 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:34.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:32.956588+0000 mon.smithi151 (mon.0) 668 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:34.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:32.977058+0000 mon.smithi151 (mon.0) 669 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi151.cywuiq", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]: dispatch 2023-11-16T13:34:34.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:32.981037+0000 mon.smithi151 (mon.0) 670 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi151.cywuiq", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]': finished 2023-11-16T13:34:34.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:32.981894+0000 mon.smithi151 (mon.0) 671 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]: dispatch 2023-11-16T13:34:34.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:32.988062+0000 mon.smithi151 (mon.0) 672 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]': finished 2023-11-16T13:34:34.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:32.989246+0000 mon.smithi151 (mon.0) 673 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:34.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:33.034074+0000 mon.smithi151 (mon.0) 674 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2023-11-16T13:34:34.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:33.037412+0000 mon.smithi151 (mon.0) 675 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2023-11-16T13:34:34.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:33.080238+0000 mon.smithi151 (mon.0) 676 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi151.cywuiq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2023-11-16T13:34:34.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:33.083322+0000 mon.smithi151 (mon.0) 677 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi151.cywuiq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2023-11-16T13:34:34.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:33.087148+0000 mon.smithi151 (mon.0) 678 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:35.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:32.929650+0000 mgr.smithi151.fcujra (mgr.14164) 211 : cephadm [INF] Saving service nfs.foo spec with placement count:2 2023-11-16T13:34:35.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:32.976716+0000 mgr.smithi151.fcujra (mgr.14164) 212 : cephadm [INF] Creating key for client.nfs.foo.0.0.smithi151.cywuiq 2023-11-16T13:34:35.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:32.981625+0000 mgr.smithi151.fcujra (mgr.14164) 213 : cephadm [INF] Ensuring nfs.foo.0 is in the ganesha grace table 2023-11-16T13:34:35.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:33.079803+0000 mgr.smithi151.fcujra (mgr.14164) 214 : cephadm [INF] Rados config object exists: conf-nfs.foo 2023-11-16T13:34:35.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:33.079937+0000 mgr.smithi151.fcujra (mgr.14164) 215 : cephadm [INF] Creating key for client.nfs.foo.0.0.smithi151.cywuiq-rgw 2023-11-16T13:34:35.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:33.088075+0000 mgr.smithi151.fcujra (mgr.14164) 216 : cephadm [INF] Deploying daemon nfs.foo.0.0.smithi151.cywuiq on smithi151 2023-11-16T13:34:35.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:33.918883+0000 mon.smithi151 (mon.0) 679 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-11-16T13:34:35.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:34.043284+0000 mon.smithi151 (mon.0) 680 : cluster [DBG] mgrmap e19: smithi151.fcujra(active, since 4m), standbys: smithi167.xxnzsz 2023-11-16T13:34:35.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:34.882077+0000 mon.smithi151 (mon.0) 681 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:35.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:34.883212+0000 mon.smithi151 (mon.0) 682 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]: dispatch 2023-11-16T13:34:35.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:34.885943+0000 mon.smithi151 (mon.0) 683 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]': finished 2023-11-16T13:34:35.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:34.886692+0000 mon.smithi151 (mon.0) 684 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]: dispatch 2023-11-16T13:34:35.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:34.891087+0000 mon.smithi151 (mon.0) 685 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]': finished 2023-11-16T13:34:35.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:34.892088+0000 mon.smithi151 (mon.0) 686 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:35.319 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:34:35.319 INFO:teuthology.orchestra.run.smithi151.stdout: "bind": "/fake", 2023-11-16T13:34:35.319 INFO:teuthology.orchestra.run.smithi151.stdout: "fs": "foofs", 2023-11-16T13:34:35.319 INFO:teuthology.orchestra.run.smithi151.stdout: "path": "/", 2023-11-16T13:34:35.319 INFO:teuthology.orchestra.run.smithi151.stdout: "cluster": "foo", 2023-11-16T13:34:35.319 INFO:teuthology.orchestra.run.smithi151.stdout: "mode": "RW" 2023-11-16T13:34:35.320 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:34:35.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:32.929650+0000 mgr.smithi151.fcujra (mgr.14164) 211 : cephadm [INF] Saving service nfs.foo spec with placement count:2 2023-11-16T13:34:35.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:32.976716+0000 mgr.smithi151.fcujra (mgr.14164) 212 : cephadm [INF] Creating key for client.nfs.foo.0.0.smithi151.cywuiq 2023-11-16T13:34:35.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:32.981625+0000 mgr.smithi151.fcujra (mgr.14164) 213 : cephadm [INF] Ensuring nfs.foo.0 is in the ganesha grace table 2023-11-16T13:34:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:33.079803+0000 mgr.smithi151.fcujra (mgr.14164) 214 : cephadm [INF] Rados config object exists: conf-nfs.foo 2023-11-16T13:34:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:33.079937+0000 mgr.smithi151.fcujra (mgr.14164) 215 : cephadm [INF] Creating key for client.nfs.foo.0.0.smithi151.cywuiq-rgw 2023-11-16T13:34:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:33.088075+0000 mgr.smithi151.fcujra (mgr.14164) 216 : cephadm [INF] Deploying daemon nfs.foo.0.0.smithi151.cywuiq on smithi151 2023-11-16T13:34:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:33.918883+0000 mon.smithi151 (mon.0) 679 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-11-16T13:34:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:34.043284+0000 mon.smithi151 (mon.0) 680 : cluster [DBG] mgrmap e19: smithi151.fcujra(active, since 4m), standbys: smithi167.xxnzsz 2023-11-16T13:34:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:34.882077+0000 mon.smithi151 (mon.0) 681 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:34.883212+0000 mon.smithi151 (mon.0) 682 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]: dispatch 2023-11-16T13:34:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:34.885943+0000 mon.smithi151 (mon.0) 683 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]': finished 2023-11-16T13:34:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:34.886692+0000 mon.smithi151 (mon.0) 684 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]: dispatch 2023-11-16T13:34:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:34.891087+0000 mon.smithi151 (mon.0) 685 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]': finished 2023-11-16T13:34:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:34.892088+0000 mon.smithi151 (mon.0) 686 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:35.691 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2023-11-16T13:34:36.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:34.145722+0000 mgr.smithi151.fcujra (mgr.14164) 217 : cluster [DBG] pgmap v169: 97 pgs: 7 creating+peering, 3 unknown, 87 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2023-11-16T13:34:36.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:34.883011+0000 mgr.smithi151.fcujra (mgr.14164) 218 : cephadm [INF] Creating key for client.nfs.foo.1.0.smithi167.pnnuhc 2023-11-16T13:34:36.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:34.886421+0000 mgr.smithi151.fcujra (mgr.14164) 219 : cephadm [INF] Ensuring nfs.foo.1 is in the ganesha grace table 2023-11-16T13:34:36.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:35.042911+0000 mon.smithi151 (mon.0) 687 : cluster [DBG] mds.? [v2:172.21.15.151:6834/3865825489,v1:172.21.15.151:6835/3865825489] up:standby 2023-11-16T13:34:36.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:35.042971+0000 mon.smithi151 (mon.0) 688 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 1 up:standby 2023-11-16T13:34:36.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:35.310399+0000 mon.smithi151 (mon.0) 689 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.foo1", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2023-11-16T13:34:36.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:35.313781+0000 mon.smithi151 (mon.0) 690 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.foo1", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2023-11-16T13:34:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:34.145722+0000 mgr.smithi151.fcujra (mgr.14164) 217 : cluster [DBG] pgmap v169: 97 pgs: 7 creating+peering, 3 unknown, 87 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2023-11-16T13:34:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:34.883011+0000 mgr.smithi151.fcujra (mgr.14164) 218 : cephadm [INF] Creating key for client.nfs.foo.1.0.smithi167.pnnuhc 2023-11-16T13:34:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:34.886421+0000 mgr.smithi151.fcujra (mgr.14164) 219 : cephadm [INF] Ensuring nfs.foo.1 is in the ganesha grace table 2023-11-16T13:34:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:35.042911+0000 mon.smithi151 (mon.0) 687 : cluster [DBG] mds.? [v2:172.21.15.151:6834/3865825489,v1:172.21.15.151:6835/3865825489] up:standby 2023-11-16T13:34:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:35.042971+0000 mon.smithi151 (mon.0) 688 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 1 up:standby 2023-11-16T13:34:36.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:35.310399+0000 mon.smithi151 (mon.0) 689 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.foo1", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2023-11-16T13:34:36.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:35.313781+0000 mon.smithi151 (mon.0) 690 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.foo1", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2023-11-16T13:34:37.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:35.287195+0000 mgr.smithi151.fcujra (mgr.14164) 220 : audit [DBG] from='client.14546 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "clusterid": "foo", "binding": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:37.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:35.287195+0000 mgr.smithi151.fcujra (mgr.14164) 220 : audit [DBG] from='client.14546 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "clusterid": "foo", "binding": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:36.146228+0000 mgr.smithi151.fcujra (mgr.14164) 221 : cluster [DBG] pgmap v170: 97 pgs: 7 creating+peering, 2 unknown, 88 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2023-11-16T13:34:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:36.986271+0000 mgr.smithi151.fcujra (mgr.14164) 222 : audit [DBG] from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:37.858597+0000 mon.smithi151 (mon.0) 691 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2023-11-16T13:34:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:37.862264+0000 mon.smithi151 (mon.0) 692 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2023-11-16T13:34:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:37.909922+0000 mgr.smithi151.fcujra (mgr.14164) 223 : cephadm [INF] Rados config object exists: conf-nfs.foo 2023-11-16T13:34:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:37.910026+0000 mgr.smithi151.fcujra (mgr.14164) 224 : cephadm [INF] Creating key for client.nfs.foo.1.0.smithi167.pnnuhc-rgw 2023-11-16T13:34:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:37.910391+0000 mon.smithi151 (mon.0) 693 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2023-11-16T13:34:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:37.913720+0000 mon.smithi151 (mon.0) 694 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2023-11-16T13:34:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:37.915274+0000 mon.smithi151 (mon.0) 695 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:38.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:37.916074+0000 mgr.smithi151.fcujra (mgr.14164) 225 : cephadm [INF] Deploying daemon nfs.foo.1.0.smithi167.pnnuhc on smithi167 2023-11-16T13:34:38.290 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:36.146228+0000 mgr.smithi151.fcujra (mgr.14164) 221 : cluster [DBG] pgmap v170: 97 pgs: 7 creating+peering, 2 unknown, 88 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2023-11-16T13:34:38.290 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:36.986271+0000 mgr.smithi151.fcujra (mgr.14164) 222 : audit [DBG] from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:38.290 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:37.858597+0000 mon.smithi151 (mon.0) 691 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2023-11-16T13:34:38.290 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:37.862264+0000 mon.smithi151 (mon.0) 692 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2023-11-16T13:34:38.290 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:37.909922+0000 mgr.smithi151.fcujra (mgr.14164) 223 : cephadm [INF] Rados config object exists: conf-nfs.foo 2023-11-16T13:34:38.290 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:37.910026+0000 mgr.smithi151.fcujra (mgr.14164) 224 : cephadm [INF] Creating key for client.nfs.foo.1.0.smithi167.pnnuhc-rgw 2023-11-16T13:34:38.290 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:37.910391+0000 mon.smithi151 (mon.0) 693 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2023-11-16T13:34:38.290 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:37.913720+0000 mon.smithi151 (mon.0) 694 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2023-11-16T13:34:38.290 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:37.915274+0000 mon.smithi151 (mon.0) 695 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:38.291 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:37.916074+0000 mgr.smithi151.fcujra (mgr.14164) 225 : cephadm [INF] Deploying daemon nfs.foo.1.0.smithi167.pnnuhc on smithi167 2023-11-16T13:34:39.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:38.146745+0000 mgr.smithi151.fcujra (mgr.14164) 226 : cluster [DBG] pgmap v171: 97 pgs: 97 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2023-11-16T13:34:39.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:38.261324+0000 mgr.smithi151.fcujra (mgr.14164) 227 : audit [DBG] from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:39.238 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:38.146745+0000 mgr.smithi151.fcujra (mgr.14164) 226 : cluster [DBG] pgmap v171: 97 pgs: 97 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2023-11-16T13:34:39.238 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:38.261324+0000 mgr.smithi151.fcujra (mgr.14164) 227 : audit [DBG] from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:39.573 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo 2/2 - 6s count:2 2023-11-16T13:34:39.946 INFO:teuthology.run_tasks:Running task vip.exec... 2023-11-16T13:34:39.956 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi151.front.sepia.ceph.com 2023-11-16T13:34:39.957 DEBUG:teuthology.orchestra.run.smithi151:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2023-11-16T13:34:39.987 INFO:teuthology.orchestra.run.smithi151.stderr:+ mkdir /mnt/foo 2023-11-16T13:34:40.000 DEBUG:teuthology.orchestra.run.smithi151:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2023-11-16T13:34:40.068 INFO:teuthology.orchestra.run.smithi151.stderr:++ hostname 2023-11-16T13:34:40.069 INFO:teuthology.orchestra.run.smithi151.stderr:+ mount -t nfs smithi151:/fake /mnt/foo -o sync 2023-11-16T13:34:40.439 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:39.432044+0000 mon.smithi151 (mon.0) 696 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:40.440 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:39.433098+0000 mon.smithi151 (mon.0) 697 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:40.440 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:39.434022+0000 mon.smithi151 (mon.0) 698 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:40.440 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:39.434670+0000 mon.smithi151 (mon.0) 699 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:40.440 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:39.555347+0000 mgr.smithi151.fcujra (mgr.14164) 228 : audit [DBG] from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:40.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:39.432044+0000 mon.smithi151 (mon.0) 696 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:40.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:39.433098+0000 mon.smithi151 (mon.0) 697 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:40.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:39.434022+0000 mon.smithi151 (mon.0) 698 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:40.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:39.434670+0000 mon.smithi151 (mon.0) 699 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:40.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:39.555347+0000 mgr.smithi151.fcujra (mgr.14164) 228 : audit [DBG] from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:40.661 DEBUG:teuthology.orchestra.run.smithi151:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2023-11-16T13:34:40.732 INFO:teuthology.orchestra.run.smithi151.stderr:+ echo test 2023-11-16T13:34:40.741 DEBUG:teuthology.orchestra.run.smithi151:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2023-11-16T13:34:40.812 INFO:teuthology.orchestra.run.smithi151.stderr:+ sync 2023-11-16T13:34:41.395 INFO:teuthology.run_tasks:Running task parallel... 2023-11-16T13:34:41.405 INFO:teuthology.task.parallel:starting parallel... 2023-11-16T13:34:41.405 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-11-16T13:34:41.406 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-11-16T13:34:41.406 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi151.front.sepia.ceph.com 2023-11-16T13:34:41.406 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-11-16T13:34:41.407 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-11-16T13:34:41.407 INFO:teuthology.task.sequential:In sequential, running task exec... 2023-11-16T13:34:41.408 INFO:teuthology.task.exec:Executing custom commands... 2023-11-16T13:34:41.408 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi151.front.sepia.ceph.com 2023-11-16T13:34:41.408 DEBUG:teuthology.orchestra.run.smithi151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2023-11-16T13:34:41.521 INFO:teuthology.orchestra.run.smithi151.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2023-11-16T13:34:41.521 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:34:41.521 INFO:teuthology.orchestra.run.smithi151.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2023-11-16T13:34:41.528 INFO:teuthology.orchestra.run.smithi151.stdout:3 of 5 processes prepared for launch 0 sec 2023-11-16T13:34:41.628 INFO:teuthology.orchestra.run.smithi151.stdout:5 of 5 processes prepared for launch 0 sec 2023-11-16T13:34:41.628 INFO:teuthology.orchestra.run.smithi151.stdout:releasing clients 2023-11-16T13:34:41.938 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:41 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:40.147368+0000 mgr.smithi151.fcujra (mgr.14164) 229 : cluster [DBG] pgmap v172: 97 pgs: 97 active+clean; 3.4 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2023-11-16T13:34:41.938 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:41 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:41.027024+0000 mon.smithi151 (mon.0) 700 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:42.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:41 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:40.147368+0000 mgr.smithi151.fcujra (mgr.14164) 229 : cluster [DBG] pgmap v172: 97 pgs: 97 active+clean; 3.4 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2023-11-16T13:34:42.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:41 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:41.027024+0000 mon.smithi151 (mon.0) 700 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:42.628 INFO:teuthology.orchestra.run.smithi151.stdout: 5 298 54.63 MB/sec warmup 1 sec latency 12.636 ms 2023-11-16T13:34:43.537 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:42.147839+0000 mgr.smithi151.fcujra (mgr.14164) 230 : cluster [DBG] pgmap v173: 97 pgs: 97 active+clean; 5.9 MiB data, 45 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 655 KiB/s wr, 14 op/s 2023-11-16T13:34:43.538 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:42.266868+0000 mon.smithi151 (mon.0) 701 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:42.147839+0000 mgr.smithi151.fcujra (mgr.14164) 230 : cluster [DBG] pgmap v173: 97 pgs: 97 active+clean; 5.9 MiB data, 45 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 655 KiB/s wr, 14 op/s 2023-11-16T13:34:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:42.266868+0000 mon.smithi151 (mon.0) 701 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:43.628 INFO:teuthology.orchestra.run.smithi151.stdout: 5 592 51.20 MB/sec warmup 2 sec latency 15.609 ms 2023-11-16T13:34:43.791 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-11-16T13:34:44.628 INFO:teuthology.orchestra.run.smithi151.stdout: 5 1674 43.70 MB/sec warmup 3 sec latency 10.977 ms 2023-11-16T13:34:45.445 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:44.122860+0000 mon.smithi151 (mon.0) 702 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:45.445 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:44.127819+0000 mon.smithi151 (mon.0) 703 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:45.445 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:44.136551+0000 mon.smithi151 (mon.0) 704 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:45.445 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:44.139551+0000 mon.smithi151 (mon.0) 705 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:45.445 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:44.141147+0000 mon.smithi151 (mon.0) 706 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:45.445 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:44.142073+0000 mon.smithi151 (mon.0) 707 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:45.445 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:44.147720+0000 mon.smithi151 (mon.0) 708 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:45.445 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:44.148807+0000 mgr.smithi151.fcujra (mgr.14164) 231 : cluster [DBG] pgmap v174: 97 pgs: 97 active+clean; 63 MiB data, 234 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 6.3 MiB/s wr, 272 op/s 2023-11-16T13:34:45.445 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:44.156092+0000 mon.smithi151 (mon.0) 709 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:44.122860+0000 mon.smithi151 (mon.0) 702 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:44.127819+0000 mon.smithi151 (mon.0) 703 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:44.136551+0000 mon.smithi151 (mon.0) 704 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:44.139551+0000 mon.smithi151 (mon.0) 705 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:44.141147+0000 mon.smithi151 (mon.0) 706 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:44.142073+0000 mon.smithi151 (mon.0) 707 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:44.147720+0000 mon.smithi151 (mon.0) 708 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:44.148807+0000 mgr.smithi151.fcujra (mgr.14164) 231 : cluster [DBG] pgmap v174: 97 pgs: 97 active+clean; 63 MiB data, 234 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 6.3 MiB/s wr, 272 op/s 2023-11-16T13:34:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:44.156092+0000 mon.smithi151 (mon.0) 709 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:45.628 INFO:teuthology.orchestra.run.smithi151.stdout: 5 3233 43.32 MB/sec warmup 4 sec latency 10.313 ms 2023-11-16T13:34:45.667 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'ceph config set global log_to_journald false --force' 2023-11-16T13:34:46.628 INFO:teuthology.orchestra.run.smithi151.stdout: 5 4352 40.54 MB/sec warmup 5 sec latency 17.479 ms 2023-11-16T13:34:47.604 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'ceph mgr module enable nfs --force' 2023-11-16T13:34:47.628 INFO:teuthology.orchestra.run.smithi151.stdout: 5 5742 38.22 MB/sec warmup 6 sec latency 12.665 ms 2023-11-16T13:34:48.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:46.149158+0000 mgr.smithi151.fcujra (mgr.14164) 232 : cluster [DBG] pgmap v175: 97 pgs: 97 active+clean; 63 MiB data, 263 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 5.4 MiB/s wr, 232 op/s 2023-11-16T13:34:48.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:47.217375+0000 mon.smithi151 (mon.0) 710 : audit [INF] from='client.? 172.21.15.151:0/3977217041' entity='client.admin' 2023-11-16T13:34:48.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:47.221669+0000 mon.smithi151 (mon.0) 711 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:48.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:47.223110+0000 mon.smithi151 (mon.0) 712 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:48.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:47.224485+0000 mon.smithi151 (mon.0) 713 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:48.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:47.230877+0000 mon.smithi151 (mon.0) 714 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:48.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:47.238368+0000 mon.smithi151 (mon.0) 715 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:48.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:47.271720+0000 mon.smithi151 (mon.0) 716 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:48.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:46.149158+0000 mgr.smithi151.fcujra (mgr.14164) 232 : cluster [DBG] pgmap v175: 97 pgs: 97 active+clean; 63 MiB data, 263 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 5.4 MiB/s wr, 232 op/s 2023-11-16T13:34:48.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:47.217375+0000 mon.smithi151 (mon.0) 710 : audit [INF] from='client.? 172.21.15.151:0/3977217041' entity='client.admin' 2023-11-16T13:34:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:47.221669+0000 mon.smithi151 (mon.0) 711 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:47.223110+0000 mon.smithi151 (mon.0) 712 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:47.224485+0000 mon.smithi151 (mon.0) 713 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:48.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:47.230877+0000 mon.smithi151 (mon.0) 714 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:48.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:47.238368+0000 mon.smithi151 (mon.0) 715 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:48.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:47.271720+0000 mon.smithi151 (mon.0) 716 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:48.628 INFO:teuthology.orchestra.run.smithi151.stdout: 5 6884 38.93 MB/sec warmup 7 sec latency 18.374 ms 2023-11-16T13:34:49.628 INFO:teuthology.orchestra.run.smithi151.stdout: 5 8011 37.46 MB/sec warmup 8 sec latency 17.218 ms 2023-11-16T13:34:49.928 INFO:teuthology.orchestra.run.smithi151.stderr:module 'nfs' is already enabled 2023-11-16T13:34:50.231 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:48.149654+0000 mgr.smithi151.fcujra (mgr.14164) 233 : cluster [DBG] pgmap v176: 97 pgs: 97 active+clean; 117 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 12 MiB/s wr, 647 op/s 2023-11-16T13:34:50.231 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:49.209904+0000 mon.smithi167 (mon.1) 16 : audit [INF] from='client.? 172.21.15.151:0/1988710869' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]: dispatch 2023-11-16T13:34:50.231 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:49.210501+0000 mon.smithi151 (mon.0) 717 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]: dispatch 2023-11-16T13:34:50.317 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-11-16T13:34:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:48.149654+0000 mgr.smithi151.fcujra (mgr.14164) 233 : cluster [DBG] pgmap v176: 97 pgs: 97 active+clean; 117 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 12 MiB/s wr, 647 op/s 2023-11-16T13:34:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:49.209904+0000 mon.smithi167 (mon.1) 16 : audit [INF] from='client.? 172.21.15.151:0/1988710869' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]: dispatch 2023-11-16T13:34:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:49.210501+0000 mon.smithi151 (mon.0) 717 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]: dispatch 2023-11-16T13:34:50.628 INFO:teuthology.orchestra.run.smithi151.stdout: 5 9391 36.37 MB/sec warmup 9 sec latency 12.578 ms 2023-11-16T13:34:51.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:49.923863+0000 mon.smithi151 (mon.0) 718 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]': finished 2023-11-16T13:34:51.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:49.923935+0000 mon.smithi151 (mon.0) 719 : cluster [DBG] mgrmap e20: smithi151.fcujra(active, since 4m), standbys: smithi167.xxnzsz 2023-11-16T13:34:51.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:49.923863+0000 mon.smithi151 (mon.0) 718 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]': finished 2023-11-16T13:34:51.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:49.923935+0000 mon.smithi151 (mon.0) 719 : cluster [DBG] mgrmap e20: smithi151.fcujra(active, since 4m), standbys: smithi167.xxnzsz 2023-11-16T13:34:51.631 INFO:teuthology.orchestra.run.smithi151.stdout: 5 10714 38.40 MB/sec warmup 10 sec latency 12.723 ms 2023-11-16T13:34:51.976 INFO:teuthology.orchestra.run.smithi151.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 2023-11-16T13:34:52.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:51 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:50.150185+0000 mgr.smithi151.fcujra (mgr.14164) 234 : cluster [DBG] pgmap v177: 97 pgs: 97 active+clean; 136 MiB data, 624 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 15 MiB/s wr, 942 op/s 2023-11-16T13:34:52.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:51 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:50.150185+0000 mgr.smithi151.fcujra (mgr.14164) 234 : cluster [DBG] pgmap v177: 97 pgs: 97 active+clean; 136 MiB data, 624 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 15 MiB/s wr, 942 op/s 2023-11-16T13:34:52.423 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-11-16T13:34:52.423 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi151.front.sepia.ceph.com 2023-11-16T13:34:52.423 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2023-11-16T13:34:52.628 INFO:teuthology.orchestra.run.smithi151.stdout: 5 11577 35.95 MB/sec warmup 11 sec latency 16.860 ms 2023-11-16T13:34:53.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:51.966333+0000 mgr.smithi151.fcujra (mgr.14164) 235 : audit [DBG] from='client.24347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ae6ce92c0812ba06c9c623ce72df4f3fc59080b4", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:53.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:51.967370+0000 mgr.smithi151.fcujra (mgr.14164) 236 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 2023-11-16T13:34:53.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:51.972194+0000 mon.smithi151 (mon.0) 720 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:53.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:51.974304+0000 mon.smithi151 (mon.0) 721 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:53.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:51.976046+0000 mon.smithi151 (mon.0) 722 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:53.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:51.977510+0000 mon.smithi151 (mon.0) 723 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:53.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:51.988154+0000 mon.smithi151 (mon.0) 724 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:53.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:52.001969+0000 mon.smithi151 (mon.0) 725 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:53.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:34:52.006480+0000 mgr.smithi151.fcujra (mgr.14164) 237 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 2023-11-16T13:34:53.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:52.150594+0000 mgr.smithi151.fcujra (mgr.14164) 238 : cluster [DBG] pgmap v178: 97 pgs: 97 active+clean; 150 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 18 MiB/s wr, 1.29k op/s 2023-11-16T13:34:53.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:51.966333+0000 mgr.smithi151.fcujra (mgr.14164) 235 : audit [DBG] from='client.24347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ae6ce92c0812ba06c9c623ce72df4f3fc59080b4", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:53.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:51.967370+0000 mgr.smithi151.fcujra (mgr.14164) 236 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 2023-11-16T13:34:53.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:51.972194+0000 mon.smithi151 (mon.0) 720 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:53.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:51.974304+0000 mon.smithi151 (mon.0) 721 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:34:53.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:51.976046+0000 mon.smithi151 (mon.0) 722 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:34:53.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:51.977510+0000 mon.smithi151 (mon.0) 723 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:34:53.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:51.988154+0000 mon.smithi151 (mon.0) 724 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:53.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:52.001969+0000 mon.smithi151 (mon.0) 725 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:34:53.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:34:52.006480+0000 mgr.smithi151.fcujra (mgr.14164) 237 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 2023-11-16T13:34:53.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:52.150594+0000 mgr.smithi151.fcujra (mgr.14164) 238 : cluster [DBG] pgmap v178: 97 pgs: 97 active+clean; 150 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 18 MiB/s wr, 1.29k op/s 2023-11-16T13:34:53.629 INFO:teuthology.orchestra.run.smithi151.stdout: 5 12829 35.05 MB/sec warmup 12 sec latency 14.035 ms 2023-11-16T13:34:54.113 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:34:54.628 INFO:teuthology.orchestra.run.smithi151.stdout: 5 13803 35.25 MB/sec warmup 13 sec latency 20.799 ms 2023-11-16T13:34:55.054 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:34:55.054 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (2m) 10s ago 4m 23.0M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:34:55.054 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (4m) 10s ago 4m 7008k - 16.2.5 6933c2a0b7dd 954bd32d4ee2 2023-11-16T13:34:55.054 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (3m) 14s ago 3m 7155k - 16.2.5 6933c2a0b7dd 649801b49aea 2023-11-16T13:34:55.054 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (2m) 10s ago 3m 29.6M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:34:55.054 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (29s) 10s ago 28s 13.3M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:34:55.055 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (27s) 14s ago 27s 17.3M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:34:55.055 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:9283 running (5m) 10s ago 5m 446M - 16.2.5 6933c2a0b7dd a009db54a97f 2023-11-16T13:34:55.055 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (3m) 14s ago 3m 376M - 16.2.5 6933c2a0b7dd a6cdf8e550f9 2023-11-16T13:34:55.055 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (5m) 10s ago 5m 96.5M 2048M 16.2.5 6933c2a0b7dd 3a504a936c5d 2023-11-16T13:34:55.055 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (3m) 14s ago 3m 90.6M 2048M 16.2.5 6933c2a0b7dd 448b57d6232a 2023-11-16T13:34:55.055 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.0.smithi151.cywuiq smithi151 *:2049 running (20s) 10s ago 20s 54.9M - 3.5 6933c2a0b7dd ecf56ff3b2ac 2023-11-16T13:34:55.055 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.0.smithi167.pnnuhc smithi167 *:2049 running (15s) 14s ago 15s 56.8M - 3.5 6933c2a0b7dd 41d900c85947 2023-11-16T13:34:55.055 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (3m) 10s ago 3m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:34:55.055 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (3m) 14s ago 3m 18.2M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:34:55.055 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (2m) 10s ago 2m 50.0M 3224M 16.2.5 6933c2a0b7dd 7598578fb701 2023-11-16T13:34:55.055 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (2m) 10s ago 2m 47.0M 3224M 16.2.5 6933c2a0b7dd 989f26649fd3 2023-11-16T13:34:55.055 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (2m) 10s ago 2m 46.8M 3224M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:34:55.056 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (2m) 10s ago 2m 49.4M 3224M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:34:55.056 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (2m) 14s ago 2m 46.1M 3992M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:34:55.056 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (109s) 14s ago 109s 44.3M 3992M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:34:55.056 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (97s) 14s ago 97s 45.4M 3992M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:34:55.056 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (84s) 14s ago 84s 49.1M 3992M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:34:55.056 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (2m) 10s ago 3m 39.3M - 2.18.1 de242295e225 336d5d209623 2023-11-16T13:34:55.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:55 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:54.091595+0000 mgr.smithi151.fcujra (mgr.14164) 239 : audit [DBG] from='client.24351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:55.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:55 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:54.151195+0000 mgr.smithi151.fcujra (mgr.14164) 240 : cluster [DBG] pgmap v179: 97 pgs: 97 active+clean; 156 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 22 MiB/s wr, 1.59k op/s 2023-11-16T13:34:55.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:55 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:54.571540+0000 mgr.smithi151.fcujra (mgr.14164) 241 : audit [DBG] from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:55.588 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:34:55.588 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:34:55.588 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:34:55.589 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:34:55.589 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:34:55.589 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:34:55.589 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:34:55.589 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:34:55.589 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2023-11-16T13:34:55.590 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:34:55.590 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:34:55.590 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:34:55.590 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:34:55.590 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:34:55.591 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 14 2023-11-16T13:34:55.591 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:34:55.591 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:34:55.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:55 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:54.091595+0000 mgr.smithi151.fcujra (mgr.14164) 239 : audit [DBG] from='client.24351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:55.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:55 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:54.151195+0000 mgr.smithi151.fcujra (mgr.14164) 240 : cluster [DBG] pgmap v179: 97 pgs: 97 active+clean; 156 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 22 MiB/s wr, 1.59k op/s 2023-11-16T13:34:55.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:55 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:54.571540+0000 mgr.smithi151.fcujra (mgr.14164) 241 : audit [DBG] from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:55.628 INFO:teuthology.orchestra.run.smithi151.stdout: 5 14583 34.77 MB/sec warmup 14 sec latency 19.374 ms 2023-11-16T13:34:56.059 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:34:56.060 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ae6ce92c0812ba06c9c623ce72df4f3fc59080b4", 2023-11-16T13:34:56.060 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:34:56.060 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [], 2023-11-16T13:34:56.060 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "", 2023-11-16T13:34:56.060 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 image" 2023-11-16T13:34:56.060 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:34:56.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:56 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:55.036001+0000 mgr.smithi151.fcujra (mgr.14164) 242 : audit [DBG] from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:56.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:56 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:55.587252+0000 mon.smithi151 (mon.0) 726 : audit [DBG] from='client.? 172.21.15.151:0/4210669184' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:34:56.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:56 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:55.036001+0000 mgr.smithi151.fcujra (mgr.14164) 242 : audit [DBG] from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:56.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:56 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:55.587252+0000 mon.smithi151 (mon.0) 726 : audit [DBG] from='client.? 172.21.15.151:0/4210669184' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:34:56.628 INFO:teuthology.orchestra.run.smithi151.stdout: 5 15348 33.16 MB/sec warmup 15 sec latency 17.480 ms 2023-11-16T13:34:57.629 INFO:teuthology.orchestra.run.smithi151.stdout: 5 16634 32.74 MB/sec warmup 16 sec latency 26.205 ms 2023-11-16T13:34:58.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:34:56.058459+0000 mgr.smithi151.fcujra (mgr.14164) 243 : audit [DBG] from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:58.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:56.151667+0000 mgr.smithi151.fcujra (mgr.14164) 244 : cluster [DBG] pgmap v180: 97 pgs: 97 active+clean; 156 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 17 MiB/s wr, 1.37k op/s 2023-11-16T13:34:58.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:34:56.058459+0000 mgr.smithi151.fcujra (mgr.14164) 243 : audit [DBG] from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:34:58.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:56.151667+0000 mgr.smithi151.fcujra (mgr.14164) 244 : cluster [DBG] pgmap v180: 97 pgs: 97 active+clean; 156 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 17 MiB/s wr, 1.37k op/s 2023-11-16T13:34:58.629 INFO:teuthology.orchestra.run.smithi151.stdout: 5 17956 34.12 MB/sec warmup 17 sec latency 13.827 ms 2023-11-16T13:34:59.629 INFO:teuthology.orchestra.run.smithi151.stdout: 5 18796 32.76 MB/sec warmup 18 sec latency 73.881 ms 2023-11-16T13:35:00.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:34:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:34:58.152168+0000 mgr.smithi151.fcujra (mgr.14164) 245 : cluster [DBG] pgmap v181: 97 pgs: 97 active+clean; 162 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 20 MiB/s wr, 1.66k op/s 2023-11-16T13:35:00.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:34:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:34:58.152168+0000 mgr.smithi151.fcujra (mgr.14164) 245 : cluster [DBG] pgmap v181: 97 pgs: 97 active+clean; 162 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 20 MiB/s wr, 1.66k op/s 2023-11-16T13:35:00.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 19783 32.14 MB/sec warmup 19 sec latency 62.145 ms 2023-11-16T13:35:01.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 20663 31.96 MB/sec warmup 20 sec latency 21.301 ms 2023-11-16T13:35:02.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:00.152860+0000 mgr.smithi151.fcujra (mgr.14164) 246 : cluster [DBG] pgmap v182: 97 pgs: 97 active+clean; 166 MiB data, 909 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 16 MiB/s wr, 1.43k op/s 2023-11-16T13:35:02.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:00.152860+0000 mgr.smithi151.fcujra (mgr.14164) 246 : cluster [DBG] pgmap v182: 97 pgs: 97 active+clean; 166 MiB data, 909 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 16 MiB/s wr, 1.43k op/s 2023-11-16T13:35:02.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 21570 32.19 MB/sec warmup 21 sec latency 22.575 ms 2023-11-16T13:35:03.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 22387 31.21 MB/sec warmup 22 sec latency 16.543 ms 2023-11-16T13:35:04.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:02.153308+0000 mgr.smithi151.fcujra (mgr.14164) 247 : cluster [DBG] pgmap v183: 97 pgs: 97 active+clean; 177 MiB data, 951 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 15 MiB/s wr, 1.46k op/s 2023-11-16T13:35:04.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:02.153308+0000 mgr.smithi151.fcujra (mgr.14164) 247 : cluster [DBG] pgmap v183: 97 pgs: 97 active+clean; 177 MiB data, 951 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 15 MiB/s wr, 1.46k op/s 2023-11-16T13:35:04.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 23566 30.87 MB/sec warmup 23 sec latency 16.577 ms 2023-11-16T13:35:05.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 24866 31.69 MB/sec warmup 24 sec latency 21.448 ms 2023-11-16T13:35:06.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:04.153894+0000 mgr.smithi151.fcujra (mgr.14164) 248 : cluster [DBG] pgmap v184: 97 pgs: 97 active+clean; 182 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 15 MiB/s wr, 1.32k op/s 2023-11-16T13:35:06.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:04.153894+0000 mgr.smithi151.fcujra (mgr.14164) 248 : cluster [DBG] pgmap v184: 97 pgs: 97 active+clean; 182 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 15 MiB/s wr, 1.32k op/s 2023-11-16T13:35:06.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 25685 31.07 MB/sec warmup 25 sec latency 19.031 ms 2023-11-16T13:35:07.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 26886 30.85 MB/sec warmup 26 sec latency 16.457 ms 2023-11-16T13:35:08.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:06.154313+0000 mgr.smithi151.fcujra (mgr.14164) 249 : cluster [DBG] pgmap v185: 97 pgs: 97 active+clean; 186 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 12 MiB/s wr, 1.13k op/s 2023-11-16T13:35:08.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:06.154313+0000 mgr.smithi151.fcujra (mgr.14164) 249 : cluster [DBG] pgmap v185: 97 pgs: 97 active+clean; 186 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 12 MiB/s wr, 1.13k op/s 2023-11-16T13:35:08.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 28041 31.17 MB/sec warmup 27 sec latency 21.706 ms 2023-11-16T13:35:09.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 28985 31.08 MB/sec warmup 28 sec latency 13.381 ms 2023-11-16T13:35:10.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:08.154775+0000 mgr.smithi151.fcujra (mgr.14164) 250 : cluster [DBG] pgmap v186: 97 pgs: 97 active+clean; 202 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 15 MiB/s wr, 1.49k op/s 2023-11-16T13:35:10.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:08.154775+0000 mgr.smithi151.fcujra (mgr.14164) 250 : cluster [DBG] pgmap v186: 97 pgs: 97 active+clean; 202 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 15 MiB/s wr, 1.49k op/s 2023-11-16T13:35:10.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 29914 30.78 MB/sec warmup 29 sec latency 14.207 ms 2023-11-16T13:35:11.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 30977 30.37 MB/sec warmup 30 sec latency 19.219 ms 2023-11-16T13:35:11.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:10.155330+0000 mgr.smithi151.fcujra (mgr.14164) 251 : cluster [DBG] pgmap v187: 97 pgs: 97 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 15 MiB/s wr, 1.47k op/s 2023-11-16T13:35:12.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:10.155330+0000 mgr.smithi151.fcujra (mgr.14164) 251 : cluster [DBG] pgmap v187: 97 pgs: 97 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 15 MiB/s wr, 1.47k op/s 2023-11-16T13:35:12.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 32123 30.99 MB/sec warmup 31 sec latency 17.842 ms 2023-11-16T13:35:13.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:12.155760+0000 mgr.smithi151.fcujra (mgr.14164) 252 : cluster [DBG] pgmap v188: 97 pgs: 97 active+clean; 209 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 15 MiB/s wr, 1.46k op/s 2023-11-16T13:35:13.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:12.155760+0000 mgr.smithi151.fcujra (mgr.14164) 252 : cluster [DBG] pgmap v188: 97 pgs: 97 active+clean; 209 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 15 MiB/s wr, 1.46k op/s 2023-11-16T13:35:13.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 32923 30.35 MB/sec warmup 32 sec latency 28.946 ms 2023-11-16T13:35:14.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 34132 30.18 MB/sec warmup 33 sec latency 24.334 ms 2023-11-16T13:35:15.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:14.156325+0000 mgr.smithi151.fcujra (mgr.14164) 253 : cluster [DBG] pgmap v189: 97 pgs: 97 active+clean; 218 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 16 MiB/s wr, 1.47k op/s 2023-11-16T13:35:15.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:14.156325+0000 mgr.smithi151.fcujra (mgr.14164) 253 : cluster [DBG] pgmap v189: 97 pgs: 97 active+clean; 218 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 16 MiB/s wr, 1.47k op/s 2023-11-16T13:35:15.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 35397 30.68 MB/sec warmup 34 sec latency 17.292 ms 2023-11-16T13:35:16.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 36316 30.38 MB/sec warmup 35 sec latency 16.195 ms 2023-11-16T13:35:17.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 37157 30.20 MB/sec warmup 36 sec latency 19.692 ms 2023-11-16T13:35:18.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:16.156774+0000 mgr.smithi151.fcujra (mgr.14164) 254 : cluster [DBG] pgmap v190: 97 pgs: 97 active+clean; 218 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 13 MiB/s wr, 1.25k op/s 2023-11-16T13:35:18.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:16.156774+0000 mgr.smithi151.fcujra (mgr.14164) 254 : cluster [DBG] pgmap v190: 97 pgs: 97 active+clean; 218 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 13 MiB/s wr, 1.25k op/s 2023-11-16T13:35:18.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 37962 29.67 MB/sec warmup 37 sec latency 29.188 ms 2023-11-16T13:35:19.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 38916 29.95 MB/sec warmup 38 sec latency 22.999 ms 2023-11-16T13:35:20.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:18.157314+0000 mgr.smithi151.fcujra (mgr.14164) 255 : cluster [DBG] pgmap v191: 97 pgs: 97 active+clean; 231 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 16 MiB/s wr, 1.51k op/s 2023-11-16T13:35:20.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:18.157314+0000 mgr.smithi151.fcujra (mgr.14164) 255 : cluster [DBG] pgmap v191: 97 pgs: 97 active+clean; 231 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 16 MiB/s wr, 1.51k op/s 2023-11-16T13:35:20.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 39633 29.67 MB/sec warmup 39 sec latency 22.978 ms 2023-11-16T13:35:21.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 40541 29.48 MB/sec warmup 40 sec latency 14.769 ms 2023-11-16T13:35:22.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:20.157939+0000 mgr.smithi151.fcujra (mgr.14164) 256 : cluster [DBG] pgmap v192: 97 pgs: 97 active+clean; 233 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 15 MiB/s wr, 1.40k op/s 2023-11-16T13:35:22.350 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:20.157939+0000 mgr.smithi151.fcujra (mgr.14164) 256 : cluster [DBG] pgmap v192: 97 pgs: 97 active+clean; 233 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 15 MiB/s wr, 1.40k op/s 2023-11-16T13:35:22.632 INFO:teuthology.orchestra.run.smithi151.stdout: 5 41705 29.32 MB/sec warmup 41 sec latency 23.149 ms 2023-11-16T13:35:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:22.120112+0000 mon.smithi151 (mon.0) 727 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]: dispatch 2023-11-16T13:35:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:22.120325+0000 mon.smithi151 (mon.0) 728 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-11-16T13:35:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:22.120496+0000 mon.smithi151 (mon.0) 729 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-11-16T13:35:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:22.120872+0000 mon.smithi151 (mon.0) 730 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-11-16T13:35:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:22.261523+0000 mon.smithi151 (mon.0) 731 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:35:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:22.266135+0000 mon.smithi151 (mon.0) 732 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:35:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:22.120112+0000 mon.smithi151 (mon.0) 727 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]: dispatch 2023-11-16T13:35:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:22.120325+0000 mon.smithi151 (mon.0) 728 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-11-16T13:35:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:22.120496+0000 mon.smithi151 (mon.0) 729 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-11-16T13:35:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:22.120872+0000 mon.smithi151 (mon.0) 730 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-11-16T13:35:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:22.261523+0000 mon.smithi151 (mon.0) 731 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:35:23.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:22.266135+0000 mon.smithi151 (mon.0) 732 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:35:23.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 42788 29.73 MB/sec warmup 42 sec latency 17.754 ms 2023-11-16T13:35:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:22.158405+0000 mgr.smithi151.fcujra (mgr.14164) 257 : cluster [DBG] pgmap v193: 97 pgs: 97 active+clean; 234 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 13 MiB/s wr, 1.20k op/s 2023-11-16T13:35:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:22.928948+0000 mon.smithi151 (mon.0) 733 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]': finished 2023-11-16T13:35:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:22.928988+0000 mon.smithi151 (mon.0) 734 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-11-16T13:35:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:22.929015+0000 mon.smithi151 (mon.0) 735 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-11-16T13:35:24.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:22.929050+0000 mon.smithi151 (mon.0) 736 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-11-16T13:35:24.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:22.929086+0000 mon.smithi151 (mon.0) 737 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-11-16T13:35:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:22.158405+0000 mgr.smithi151.fcujra (mgr.14164) 257 : cluster [DBG] pgmap v193: 97 pgs: 97 active+clean; 234 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 13 MiB/s wr, 1.20k op/s 2023-11-16T13:35:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:22.928948+0000 mon.smithi151 (mon.0) 733 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]': finished 2023-11-16T13:35:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:22.928988+0000 mon.smithi151 (mon.0) 734 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-11-16T13:35:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:22.929015+0000 mon.smithi151 (mon.0) 735 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-11-16T13:35:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:22.929050+0000 mon.smithi151 (mon.0) 736 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-11-16T13:35:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:22.929086+0000 mon.smithi151 (mon.0) 737 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-11-16T13:35:24.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 43619 29.29 MB/sec warmup 43 sec latency 21.399 ms 2023-11-16T13:35:25.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:23.933667+0000 mon.smithi151 (mon.0) 738 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-11-16T13:35:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:24.158909+0000 mgr.smithi151.fcujra (mgr.14164) 258 : cluster [DBG] pgmap v196: 97 pgs: 1 peering, 96 active+clean; 243 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 16 MiB/s wr, 1.47k op/s 2023-11-16T13:35:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:23.933667+0000 mon.smithi151 (mon.0) 738 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-11-16T13:35:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:24.158909+0000 mgr.smithi151.fcujra (mgr.14164) 258 : cluster [DBG] pgmap v196: 97 pgs: 1 peering, 96 active+clean; 243 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 16 MiB/s wr, 1.47k op/s 2023-11-16T13:35:25.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 44966 29.23 MB/sec warmup 44 sec latency 16.704 ms 2023-11-16T13:35:26.541 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:35:26.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 46089 29.64 MB/sec warmup 45 sec latency 20.603 ms 2023-11-16T13:35:27.353 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:35:27.354 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (3m) 43s ago 4m 23.0M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:35:27.354 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (4m) 43s ago 4m 7008k - 16.2.5 6933c2a0b7dd 954bd32d4ee2 2023-11-16T13:35:27.354 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (3m) 46s ago 3m 7155k - 16.2.5 6933c2a0b7dd 649801b49aea 2023-11-16T13:35:27.354 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (3m) 43s ago 4m 29.6M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:35:27.354 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (61s) 43s ago 61s 13.3M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:35:27.354 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (60s) 46s ago 59s 17.3M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:35:27.354 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:9283 running (5m) 43s ago 5m 446M - 16.2.5 6933c2a0b7dd a009db54a97f 2023-11-16T13:35:27.354 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (3m) 46s ago 3m 376M - 16.2.5 6933c2a0b7dd a6cdf8e550f9 2023-11-16T13:35:27.354 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (5m) 43s ago 5m 96.5M 2048M 16.2.5 6933c2a0b7dd 3a504a936c5d 2023-11-16T13:35:27.354 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (3m) 46s ago 3m 90.6M 2048M 16.2.5 6933c2a0b7dd 448b57d6232a 2023-11-16T13:35:27.355 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.0.smithi151.cywuiq smithi151 *:2049 running (52s) 43s ago 52s 54.9M - 3.5 6933c2a0b7dd ecf56ff3b2ac 2023-11-16T13:35:27.355 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.0.smithi167.pnnuhc smithi167 *:2049 running (48s) 46s ago 47s 56.8M - 3.5 6933c2a0b7dd 41d900c85947 2023-11-16T13:35:27.355 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (4m) 43s ago 4m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:35:27.355 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (3m) 46s ago 3m 18.2M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:35:27.355 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (3m) 43s ago 3m 50.0M 3224M 16.2.5 6933c2a0b7dd 7598578fb701 2023-11-16T13:35:27.355 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (3m) 43s ago 3m 47.0M 3224M 16.2.5 6933c2a0b7dd 989f26649fd3 2023-11-16T13:35:27.355 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (2m) 43s ago 2m 46.8M 3224M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:35:27.355 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (2m) 43s ago 2m 49.4M 3224M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:35:27.355 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (2m) 46s ago 2m 46.1M 3992M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:35:27.355 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (2m) 46s ago 2m 44.3M 3992M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:35:27.355 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (2m) 46s ago 2m 45.4M 3992M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:35:27.355 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (116s) 46s ago 116s 49.1M 3992M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:35:27.356 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (3m) 43s ago 4m 39.3M - 2.18.1 de242295e225 336d5d209623 2023-11-16T13:35:27.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 46726 29.31 MB/sec warmup 46 sec latency 62.364 ms 2023-11-16T13:35:27.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:26.159472+0000 mgr.smithi151.fcujra (mgr.14164) 259 : cluster [DBG] pgmap v197: 97 pgs: 1 peering, 96 active+clean; 251 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 12 MiB/s wr, 1.16k op/s 2023-11-16T13:35:27.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:26.381651+0000 mon.smithi151 (mon.0) 739 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:35:27.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:35:26.382218+0000 mgr.smithi151.fcujra (mgr.14164) 260 : cephadm [INF] Upgrade: Target is version 17.2.7-68-gae6ce92c (unknown) 2023-11-16T13:35:27.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:35:26.382294+0000 mgr.smithi151.fcujra (mgr.14164) 261 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b, digests ['quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b'] 2023-11-16T13:35:27.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:26.383558+0000 mon.smithi151 (mon.0) 740 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:35:27.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:35:26.384597+0000 mgr.smithi151.fcujra (mgr.14164) 262 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi151.fcujra) 2023-11-16T13:35:27.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:26.522856+0000 mgr.smithi151.fcujra (mgr.14164) 263 : audit [DBG] from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:35:27.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:35:26.672010+0000 mgr.smithi151.fcujra (mgr.14164) 264 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b on smithi167 2023-11-16T13:35:27.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:26.159472+0000 mgr.smithi151.fcujra (mgr.14164) 259 : cluster [DBG] pgmap v197: 97 pgs: 1 peering, 96 active+clean; 251 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 12 MiB/s wr, 1.16k op/s 2023-11-16T13:35:27.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:26.381651+0000 mon.smithi151 (mon.0) 739 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:35:27.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:35:26.382218+0000 mgr.smithi151.fcujra (mgr.14164) 260 : cephadm [INF] Upgrade: Target is version 17.2.7-68-gae6ce92c (unknown) 2023-11-16T13:35:27.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:35:26.382294+0000 mgr.smithi151.fcujra (mgr.14164) 261 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b, digests ['quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b'] 2023-11-16T13:35:27.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:26.383558+0000 mon.smithi151 (mon.0) 740 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:35:27.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:35:26.384597+0000 mgr.smithi151.fcujra (mgr.14164) 262 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi151.fcujra) 2023-11-16T13:35:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:26.522856+0000 mgr.smithi151.fcujra (mgr.14164) 263 : audit [DBG] from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:35:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:35:26.672010+0000 mgr.smithi151.fcujra (mgr.14164) 264 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b on smithi167 2023-11-16T13:35:27.847 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:35:27.847 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:35:27.847 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:35:27.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:35:27.847 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:35:27.848 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:35:27.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:35:27.848 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:35:27.848 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2023-11-16T13:35:27.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:35:27.848 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:35:27.848 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:35:27.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:35:27.848 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:35:27.848 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 14 2023-11-16T13:35:27.848 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:35:27.848 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:35:28.259 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:35:28.260 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:35:28.260 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:35:28.260 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [], 2023-11-16T13:35:28.260 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "0/23 ceph daemons upgraded", 2023-11-16T13:35:28.260 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b image on host smithi167" 2023-11-16T13:35:28.260 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:35:28.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 47522 29.09 MB/sec warmup 47 sec latency 78.930 ms 2023-11-16T13:35:28.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:26.951528+0000 mgr.smithi151.fcujra (mgr.14164) 265 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:35:28.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:27.340213+0000 mgr.smithi151.fcujra (mgr.14164) 266 : audit [DBG] from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:35:28.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:27.845934+0000 mon.smithi151 (mon.0) 741 : audit [DBG] from='client.? 172.21.15.151:0/3062094664' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:35:28.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:26.951528+0000 mgr.smithi151.fcujra (mgr.14164) 265 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:35:28.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:27.340213+0000 mgr.smithi151.fcujra (mgr.14164) 266 : audit [DBG] from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:35:28.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:27.845934+0000 mon.smithi151 (mon.0) 741 : audit [DBG] from='client.? 172.21.15.151:0/3062094664' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:35:29.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 48622 28.86 MB/sec warmup 48 sec latency 26.141 ms 2023-11-16T13:35:29.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:28.159986+0000 mgr.smithi151.fcujra (mgr.14164) 267 : cluster [DBG] pgmap v198: 97 pgs: 1 peering, 96 active+clean; 252 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 13 MiB/s wr, 1.03k op/s 2023-11-16T13:35:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:28.258231+0000 mgr.smithi151.fcujra (mgr.14164) 268 : audit [DBG] from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:35:29.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:28.159986+0000 mgr.smithi151.fcujra (mgr.14164) 267 : cluster [DBG] pgmap v198: 97 pgs: 1 peering, 96 active+clean; 252 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 13 MiB/s wr, 1.03k op/s 2023-11-16T13:35:29.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:28.258231+0000 mgr.smithi151.fcujra (mgr.14164) 268 : audit [DBG] from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:35:30.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 49814 29.31 MB/sec warmup 49 sec latency 30.628 ms 2023-11-16T13:35:31.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 50750 29.02 MB/sec warmup 50 sec latency 20.236 ms 2023-11-16T13:35:32.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:30.160489+0000 mgr.smithi151.fcujra (mgr.14164) 269 : cluster [DBG] pgmap v199: 97 pgs: 97 active+clean; 263 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 15 MiB/s wr, 1.39k op/s 2023-11-16T13:35:32.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:30.160489+0000 mgr.smithi151.fcujra (mgr.14164) 269 : cluster [DBG] pgmap v199: 97 pgs: 97 active+clean; 263 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 15 MiB/s wr, 1.39k op/s 2023-11-16T13:35:32.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 51884 28.92 MB/sec warmup 51 sec latency 28.554 ms 2023-11-16T13:35:33.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 52596 28.86 MB/sec warmup 52 sec latency 25.870 ms 2023-11-16T13:35:34.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:32.160929+0000 mgr.smithi151.fcujra (mgr.14164) 270 : cluster [DBG] pgmap v200: 97 pgs: 97 active+clean; 271 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 15 MiB/s wr, 1.49k op/s 2023-11-16T13:35:34.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:32.160929+0000 mgr.smithi151.fcujra (mgr.14164) 270 : cluster [DBG] pgmap v200: 97 pgs: 97 active+clean; 271 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 15 MiB/s wr, 1.49k op/s 2023-11-16T13:35:34.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 53212 28.78 MB/sec warmup 53 sec latency 30.563 ms 2023-11-16T13:35:35.634 INFO:teuthology.orchestra.run.smithi151.stdout: 5 53735 28.50 MB/sec warmup 54 sec latency 33.920 ms 2023-11-16T13:35:36.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:34.161426+0000 mgr.smithi151.fcujra (mgr.14164) 271 : cluster [DBG] pgmap v201: 97 pgs: 97 active+clean; 275 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 15 MiB/s wr, 1.39k op/s 2023-11-16T13:35:36.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:34.161426+0000 mgr.smithi151.fcujra (mgr.14164) 271 : cluster [DBG] pgmap v201: 97 pgs: 97 active+clean; 275 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 15 MiB/s wr, 1.39k op/s 2023-11-16T13:35:36.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 54536 28.28 MB/sec warmup 55 sec latency 39.106 ms 2023-11-16T13:35:37.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 55322 28.01 MB/sec warmup 56 sec latency 42.550 ms 2023-11-16T13:35:38.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:36.161850+0000 mgr.smithi151.fcujra (mgr.14164) 272 : cluster [DBG] pgmap v202: 97 pgs: 97 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 12 MiB/s wr, 1.16k op/s 2023-11-16T13:35:38.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:37 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:37.279515+0000 mon.smithi151 (mon.0) 742 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:35:38.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:36.161850+0000 mgr.smithi151.fcujra (mgr.14164) 272 : cluster [DBG] pgmap v202: 97 pgs: 97 active+clean; 275 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 12 MiB/s wr, 1.16k op/s 2023-11-16T13:35:38.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:37.279515+0000 mon.smithi151 (mon.0) 742 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:35:38.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 55858 27.79 MB/sec warmup 57 sec latency 30.400 ms 2023-11-16T13:35:39.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 57015 28.13 MB/sec warmup 58 sec latency 13.004 ms 2023-11-16T13:35:40.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:39 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:38.162293+0000 mgr.smithi151.fcujra (mgr.14164) 273 : cluster [DBG] pgmap v203: 97 pgs: 97 active+clean; 282 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 14 MiB/s wr, 1.26k op/s 2023-11-16T13:35:40.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:39 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:38.162293+0000 mgr.smithi151.fcujra (mgr.14164) 273 : cluster [DBG] pgmap v203: 97 pgs: 97 active+clean; 282 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 14 MiB/s wr, 1.26k op/s 2023-11-16T13:35:40.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 57823 27.83 MB/sec warmup 59 sec latency 13.501 ms 2023-11-16T13:35:41.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 58576 27.69 MB/sec warmup 60 sec latency 57.621 ms 2023-11-16T13:35:42.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:41 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:40.163314+0000 mgr.smithi151.fcujra (mgr.14164) 274 : cluster [DBG] pgmap v204: 97 pgs: 97 active+clean; 286 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 13 MiB/s wr, 1.26k op/s 2023-11-16T13:35:42.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:41 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:40.163314+0000 mgr.smithi151.fcujra (mgr.14164) 274 : cluster [DBG] pgmap v204: 97 pgs: 97 active+clean; 286 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 13 MiB/s wr, 1.26k op/s 2023-11-16T13:35:42.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 59183 27.38 MB/sec warmup 61 sec latency 25.432 ms 2023-11-16T13:35:43.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:42.163709+0000 mgr.smithi151.fcujra (mgr.14164) 275 : cluster [DBG] pgmap v205: 97 pgs: 97 active+clean; 282 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.09k op/s 2023-11-16T13:35:43.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:42.163709+0000 mgr.smithi151.fcujra (mgr.14164) 275 : cluster [DBG] pgmap v205: 97 pgs: 97 active+clean; 282 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.09k op/s 2023-11-16T13:35:43.633 INFO:teuthology.orchestra.run.smithi151.stdout: 5 59683 27.29 MB/sec warmup 62 sec latency 30.629 ms 2023-11-16T13:35:44.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 60582 27.43 MB/sec warmup 63 sec latency 30.839 ms 2023-11-16T13:35:45.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:44.164243+0000 mgr.smithi151.fcujra (mgr.14164) 276 : cluster [DBG] pgmap v206: 97 pgs: 97 active+clean; 293 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.18k op/s 2023-11-16T13:35:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:44.164243+0000 mgr.smithi151.fcujra (mgr.14164) 276 : cluster [DBG] pgmap v206: 97 pgs: 97 active+clean; 293 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.18k op/s 2023-11-16T13:35:45.634 INFO:teuthology.orchestra.run.smithi151.stdout: 5 61224 27.10 MB/sec warmup 64 sec latency 30.707 ms 2023-11-16T13:35:46.634 INFO:teuthology.orchestra.run.smithi151.stdout: 5 61752 26.92 MB/sec warmup 65 sec latency 30.284 ms 2023-11-16T13:35:47.634 INFO:teuthology.orchestra.run.smithi151.stdout: 5 62773 26.77 MB/sec warmup 66 sec latency 25.061 ms 2023-11-16T13:35:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:47 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:46.164652+0000 mgr.smithi151.fcujra (mgr.14164) 277 : cluster [DBG] pgmap v207: 97 pgs: 97 active+clean; 293 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 8.8 MiB/s wr, 911 op/s 2023-11-16T13:35:48.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:46.164652+0000 mgr.smithi151.fcujra (mgr.14164) 277 : cluster [DBG] pgmap v207: 97 pgs: 97 active+clean; 293 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 8.8 MiB/s wr, 911 op/s 2023-11-16T13:35:48.639 INFO:teuthology.orchestra.run.smithi151.stdout: 5 63806 27.03 MB/sec warmup 67 sec latency 18.253 ms 2023-11-16T13:35:49.634 INFO:teuthology.orchestra.run.smithi151.stdout: 5 64458 26.86 MB/sec warmup 68 sec latency 23.340 ms 2023-11-16T13:35:50.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:48.165152+0000 mgr.smithi151.fcujra (mgr.14164) 278 : cluster [DBG] pgmap v208: 97 pgs: 97 active+clean; 302 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.16k op/s 2023-11-16T13:35:50.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:48.165152+0000 mgr.smithi151.fcujra (mgr.14164) 278 : cluster [DBG] pgmap v208: 97 pgs: 97 active+clean; 302 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.16k op/s 2023-11-16T13:35:50.634 INFO:teuthology.orchestra.run.smithi151.stdout: 5 64917 26.57 MB/sec warmup 69 sec latency 30.550 ms 2023-11-16T13:35:51.634 INFO:teuthology.orchestra.run.smithi151.stdout: 5 65574 26.46 MB/sec warmup 70 sec latency 38.106 ms 2023-11-16T13:35:52.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:51 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:50.165607+0000 mgr.smithi151.fcujra (mgr.14164) 279 : cluster [DBG] pgmap v209: 97 pgs: 97 active+clean; 310 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2023-11-16T13:35:52.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:51 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:50.165607+0000 mgr.smithi151.fcujra (mgr.14164) 279 : cluster [DBG] pgmap v209: 97 pgs: 97 active+clean; 310 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2023-11-16T13:35:52.634 INFO:teuthology.orchestra.run.smithi151.stdout: 5 66362 26.28 MB/sec warmup 71 sec latency 21.143 ms 2023-11-16T13:35:53.634 INFO:teuthology.orchestra.run.smithi151.stdout: 5 67194 26.37 MB/sec warmup 72 sec latency 43.608 ms 2023-11-16T13:35:54.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:52.166084+0000 mgr.smithi151.fcujra (mgr.14164) 280 : cluster [DBG] pgmap v210: 97 pgs: 97 active+clean; 317 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.13k op/s 2023-11-16T13:35:54.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:53 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:52.166084+0000 mgr.smithi151.fcujra (mgr.14164) 280 : cluster [DBG] pgmap v210: 97 pgs: 97 active+clean; 317 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.13k op/s 2023-11-16T13:35:54.634 INFO:teuthology.orchestra.run.smithi151.stdout: 5 67752 26.31 MB/sec warmup 73 sec latency 23.007 ms 2023-11-16T13:35:55.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 68186 25.99 MB/sec warmup 74 sec latency 43.763 ms 2023-11-16T13:35:56.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:55 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:54.166600+0000 mgr.smithi151.fcujra (mgr.14164) 281 : cluster [DBG] pgmap v211: 97 pgs: 97 active+clean; 317 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.14k op/s 2023-11-16T13:35:56.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:55 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:54.166600+0000 mgr.smithi151.fcujra (mgr.14164) 281 : cluster [DBG] pgmap v211: 97 pgs: 97 active+clean; 317 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.14k op/s 2023-11-16T13:35:56.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 68843 25.88 MB/sec warmup 75 sec latency 23.517 ms 2023-11-16T13:35:57.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 69829 25.75 MB/sec warmup 76 sec latency 22.398 ms 2023-11-16T13:35:58.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:57 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:56.167047+0000 mgr.smithi151.fcujra (mgr.14164) 282 : cluster [DBG] pgmap v212: 97 pgs: 97 active+clean; 317 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 8.7 MiB/s wr, 794 op/s 2023-11-16T13:35:58.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:56.167047+0000 mgr.smithi151.fcujra (mgr.14164) 282 : cluster [DBG] pgmap v212: 97 pgs: 97 active+clean; 317 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 8.7 MiB/s wr, 794 op/s 2023-11-16T13:35:58.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 70571 25.82 MB/sec warmup 77 sec latency 70.487 ms 2023-11-16T13:35:58.704 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:35:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:35:58.167580+0000 mgr.smithi151.fcujra (mgr.14164) 283 : cluster [DBG] pgmap v213: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2023-11-16T13:35:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:58.682505+0000 mgr.smithi151.fcujra (mgr.14164) 284 : audit [DBG] from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:35:59.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:35:58.167580+0000 mgr.smithi151.fcujra (mgr.14164) 283 : cluster [DBG] pgmap v213: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2023-11-16T13:35:59.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:58.682505+0000 mgr.smithi151.fcujra (mgr.14164) 284 : audit [DBG] from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:35:59.535 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:35:59.535 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (4m) 75s ago 5m 23.0M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:35:59.535 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (5m) 75s ago 5m 7008k - 16.2.5 6933c2a0b7dd 954bd32d4ee2 2023-11-16T13:35:59.536 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (4m) 78s ago 4m 7155k - 16.2.5 6933c2a0b7dd 649801b49aea 2023-11-16T13:35:59.536 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (3m) 75s ago 4m 29.6M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:35:59.536 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (93s) 75s ago 93s 13.3M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:35:59.536 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (92s) 78s ago 91s 17.3M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:35:59.536 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:9283 running (6m) 75s ago 6m 446M - 16.2.5 6933c2a0b7dd a009db54a97f 2023-11-16T13:35:59.536 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (4m) 78s ago 4m 376M - 16.2.5 6933c2a0b7dd a6cdf8e550f9 2023-11-16T13:35:59.536 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (6m) 75s ago 6m 96.5M 2048M 16.2.5 6933c2a0b7dd 3a504a936c5d 2023-11-16T13:35:59.536 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (4m) 78s ago 4m 90.6M 2048M 16.2.5 6933c2a0b7dd 448b57d6232a 2023-11-16T13:35:59.536 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.0.smithi151.cywuiq smithi151 *:2049 running (85s) 75s ago 84s 54.9M - 3.5 6933c2a0b7dd ecf56ff3b2ac 2023-11-16T13:35:59.536 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.0.smithi167.pnnuhc smithi167 *:2049 running (80s) 78s ago 80s 56.8M - 3.5 6933c2a0b7dd 41d900c85947 2023-11-16T13:35:59.536 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (4m) 75s ago 4m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:35:59.536 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (4m) 78s ago 4m 18.2M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:35:59.537 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (4m) 75s ago 4m 50.0M 3224M 16.2.5 6933c2a0b7dd 7598578fb701 2023-11-16T13:35:59.537 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (3m) 75s ago 3m 47.0M 3224M 16.2.5 6933c2a0b7dd 989f26649fd3 2023-11-16T13:35:59.537 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (3m) 75s ago 3m 46.8M 3224M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:35:59.537 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (3m) 75s ago 3m 49.4M 3224M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:35:59.537 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (3m) 78s ago 3m 46.1M 3992M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:35:59.537 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (2m) 78s ago 2m 44.3M 3992M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:35:59.537 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (2m) 78s ago 2m 45.4M 3992M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:35:59.537 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (2m) 78s ago 2m 49.1M 3992M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:35:59.537 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (3m) 75s ago 4m 39.3M - 2.18.1 de242295e225 336d5d209623 2023-11-16T13:35:59.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 71210 25.83 MB/sec warmup 78 sec latency 22.931 ms 2023-11-16T13:35:59.938 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:35:59.939 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:35:59.939 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:35:59.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:35:59.939 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:35:59.939 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:35:59.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:35:59.939 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:35:59.939 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2023-11-16T13:35:59.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:35:59.939 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:35:59.939 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:35:59.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:35:59.940 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:35:59.940 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 14 2023-11-16T13:35:59.940 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:35:59.940 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:36:00.256 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:36:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:36:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:36:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [], 2023-11-16T13:36:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "0/23 ceph daemons upgraded", 2023-11-16T13:36:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b image on host smithi167" 2023-11-16T13:36:00.256 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:36:00.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:59.115979+0000 mgr.smithi151.fcujra (mgr.14164) 285 : audit [DBG] from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:00.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:59.520822+0000 mgr.smithi151.fcujra (mgr.14164) 286 : audit [DBG] from='client.24395 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:00.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:35:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:35:59.937677+0000 mon.smithi151 (mon.0) 743 : audit [DBG] from='client.? 172.21.15.151:0/1849653468' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:36:00.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:59.115979+0000 mgr.smithi151.fcujra (mgr.14164) 285 : audit [DBG] from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:00.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:59.520822+0000 mgr.smithi151.fcujra (mgr.14164) 286 : audit [DBG] from='client.24395 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:00.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:35:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:35:59.937677+0000 mon.smithi151 (mon.0) 743 : audit [DBG] from='client.? 172.21.15.151:0/1849653468' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:36:00.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 71576 25.54 MB/sec warmup 79 sec latency 34.959 ms 2023-11-16T13:36:01.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 71939 25.28 MB/sec warmup 80 sec latency 38.273 ms 2023-11-16T13:36:02.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:00.168062+0000 mgr.smithi151.fcujra (mgr.14164) 287 : cluster [DBG] pgmap v214: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2023-11-16T13:36:02.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:00.254772+0000 mgr.smithi151.fcujra (mgr.14164) 288 : audit [DBG] from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:02.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:00.168062+0000 mgr.smithi151.fcujra (mgr.14164) 287 : cluster [DBG] pgmap v214: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2023-11-16T13:36:02.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:01 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:00.254772+0000 mgr.smithi151.fcujra (mgr.14164) 288 : audit [DBG] from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:02.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 72567 25.22 MB/sec warmup 81 sec latency 32.023 ms 2023-11-16T13:36:03.413 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:02.151933+0000 mgr.smithi151.fcujra (mgr.14164) 289 : cephadm [INF] Upgrade: Updating mgr.smithi167.xxnzsz 2023-11-16T13:36:03.414 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:02.156745+0000 mon.smithi151 (mon.0) 744 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:36:03.414 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:02.157473+0000 mon.smithi151 (mon.0) 745 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi167.xxnzsz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:36:03.414 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:02.158097+0000 mon.smithi151 (mon.0) 746 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:36:03.414 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:02.158718+0000 mon.smithi151 (mon.0) 747 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:03.414 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:02.159451+0000 mgr.smithi151.fcujra (mgr.14164) 290 : cephadm [INF] Deploying daemon mgr.smithi167.xxnzsz on smithi167 2023-11-16T13:36:03.414 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:02.168478+0000 mgr.smithi151.fcujra (mgr.14164) 291 : cluster [DBG] pgmap v215: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.3 MiB/s wr, 913 op/s 2023-11-16T13:36:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:02.151933+0000 mgr.smithi151.fcujra (mgr.14164) 289 : cephadm [INF] Upgrade: Updating mgr.smithi167.xxnzsz 2023-11-16T13:36:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:02.156745+0000 mon.smithi151 (mon.0) 744 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:36:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:02.157473+0000 mon.smithi151 (mon.0) 745 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi167.xxnzsz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:36:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:02.158097+0000 mon.smithi151 (mon.0) 746 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:36:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:02.158718+0000 mon.smithi151 (mon.0) 747 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:02.159451+0000 mgr.smithi151.fcujra (mgr.14164) 290 : cephadm [INF] Deploying daemon mgr.smithi167.xxnzsz on smithi167 2023-11-16T13:36:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:02.168478+0000 mgr.smithi151.fcujra (mgr.14164) 291 : cluster [DBG] pgmap v215: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.3 MiB/s wr, 913 op/s 2023-11-16T13:36:03.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 73317 25.00 MB/sec warmup 82 sec latency 33.608 ms 2023-11-16T13:36:04.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 73826 24.95 MB/sec warmup 83 sec latency 43.386 ms 2023-11-16T13:36:05.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:04.088156+0000 mon.smithi151 (mon.0) 748 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:36:05.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:04.088823+0000 mon.smithi151 (mon.0) 749 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:36:05.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:04.089663+0000 mon.smithi151 (mon.0) 750 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:05.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:04.090336+0000 mon.smithi151 (mon.0) 751 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:36:05.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:04.168956+0000 mgr.smithi151.fcujra (mgr.14164) 292 : cluster [DBG] pgmap v216: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 971 op/s 2023-11-16T13:36:05.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:04.088156+0000 mon.smithi151 (mon.0) 748 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:36:05.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:04.088823+0000 mon.smithi151 (mon.0) 749 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:36:05.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:04.089663+0000 mon.smithi151 (mon.0) 750 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:05.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:04.090336+0000 mon.smithi151 (mon.0) 751 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:36:05.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:04.168956+0000 mgr.smithi151.fcujra (mgr.14164) 292 : cluster [DBG] pgmap v216: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 971 op/s 2023-11-16T13:36:05.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 74762 25.11 MB/sec warmup 84 sec latency 19.026 ms 2023-11-16T13:36:06.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 75419 24.91 MB/sec warmup 85 sec latency 42.599 ms 2023-11-16T13:36:07.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 75962 24.80 MB/sec warmup 86 sec latency 34.215 ms 2023-11-16T13:36:07.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:06.169349+0000 mgr.smithi151.fcujra (mgr.14164) 293 : cluster [DBG] pgmap v217: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 7.7 MiB/s wr, 861 op/s 2023-11-16T13:36:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:06.438174+0000 mon.smithi151 (mon.0) 752 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:36:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:06.765488+0000 mon.smithi151 (mon.0) 753 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:36:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:06.774219+0000 mon.smithi151 (mon.0) 754 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:36:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:06.777090+0000 mon.smithi151 (mon.0) 755 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:36:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:06.778577+0000 mgr.smithi151.fcujra (mgr.14164) 294 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi151.fcujra) 2023-11-16T13:36:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:06.781895+0000 mgr.smithi151.fcujra (mgr.14164) 295 : cephadm [INF] Failing over to other MGR 2023-11-16T13:36:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:06.782372+0000 mon.smithi151 (mon.0) 756 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr fail", "who": "smithi151.fcujra"}]: dispatch 2023-11-16T13:36:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:06.786326+0000 mon.smithi151 (mon.0) 757 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-11-16T13:36:07.811 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:06.169349+0000 mgr.smithi151.fcujra (mgr.14164) 293 : cluster [DBG] pgmap v217: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 7.7 MiB/s wr, 861 op/s 2023-11-16T13:36:07.811 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:06.438174+0000 mon.smithi151 (mon.0) 752 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:36:07.812 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:06.765488+0000 mon.smithi151 (mon.0) 753 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:36:07.812 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:06.774219+0000 mon.smithi151 (mon.0) 754 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' 2023-11-16T13:36:07.812 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:06.777090+0000 mon.smithi151 (mon.0) 755 : audit [DBG] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:36:07.812 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:06.778577+0000 mgr.smithi151.fcujra (mgr.14164) 294 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi151.fcujra) 2023-11-16T13:36:07.812 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:06.781895+0000 mgr.smithi151.fcujra (mgr.14164) 295 : cephadm [INF] Failing over to other MGR 2023-11-16T13:36:07.812 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:06.782372+0000 mon.smithi151 (mon.0) 756 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr fail", "who": "smithi151.fcujra"}]: dispatch 2023-11-16T13:36:07.812 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:06.786326+0000 mon.smithi151 (mon.0) 757 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-11-16T13:36:08.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 76609 24.62 MB/sec warmup 87 sec latency 36.331 ms 2023-11-16T13:36:08.724 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:07.527516+0000 mon.smithi151 (mon.0) 758 : cluster [DBG] Standby manager daemon smithi151.fcujra started 2023-11-16T13:36:08.725 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:07.779735+0000 mon.smithi151 (mon.0) 759 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "mgr fail", "who": "smithi151.fcujra"}]': finished 2023-11-16T13:36:08.725 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:07.779792+0000 mon.smithi151 (mon.0) 760 : cluster [DBG] mgrmap e21: smithi167.xxnzsz(active, starting, since 0.996535s), standbys: smithi151.fcujra 2023-11-16T13:36:08.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:08 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:07.527516+0000 mon.smithi151 (mon.0) 758 : cluster [DBG] Standby manager daemon smithi151.fcujra started 2023-11-16T13:36:08.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:08 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:07.779735+0000 mon.smithi151 (mon.0) 759 : audit [INF] from='mgr.14164 172.21.15.151:0/1527459394' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "mgr fail", "who": "smithi151.fcujra"}]': finished 2023-11-16T13:36:08.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:08 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:07.779792+0000 mon.smithi151 (mon.0) 760 : cluster [DBG] mgrmap e21: smithi167.xxnzsz(active, starting, since 0.996535s), standbys: smithi151.fcujra 2023-11-16T13:36:09.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 77009 24.43 MB/sec warmup 88 sec latency 32.615 ms 2023-11-16T13:36:10.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 77551 24.43 MB/sec warmup 89 sec latency 31.687 ms 2023-11-16T13:36:11.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 78083 24.40 MB/sec warmup 90 sec latency 47.390 ms 2023-11-16T13:36:12.635 INFO:teuthology.orchestra.run.smithi151.stdout: 5 78407 24.26 MB/sec warmup 91 sec latency 39.736 ms 2023-11-16T13:36:13.495 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:13.157766+0000 mon.smithi151 (mon.0) 761 : cluster [INF] Active manager daemon smithi167.xxnzsz restarted 2023-11-16T13:36:13.495 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:13.158777+0000 mon.smithi151 (mon.0) 762 : cluster [INF] Activating manager daemon smithi167.xxnzsz 2023-11-16T13:36:13.495 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.160226+0000 mon.smithi151 (mon.0) 763 : audit [DBG] from='mgr.? 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/crt"}]: dispatch 2023-11-16T13:36:13.495 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.161354+0000 mon.smithi151 (mon.0) 764 : audit [DBG] from='mgr.? 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-16T13:36:13.496 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:13.165308+0000 mon.smithi151 (mon.0) 765 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-11-16T13:36:13.496 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.165468+0000 mon.smithi151 (mon.0) 766 : audit [DBG] from='mgr.? 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/key"}]: dispatch 2023-11-16T13:36:13.496 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.166156+0000 mon.smithi151 (mon.0) 767 : audit [DBG] from='mgr.? 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:36:13.500 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:13.157766+0000 mon.smithi151 (mon.0) 761 : cluster [INF] Active manager daemon smithi167.xxnzsz restarted 2023-11-16T13:36:13.500 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:13.158777+0000 mon.smithi151 (mon.0) 762 : cluster [INF] Activating manager daemon smithi167.xxnzsz 2023-11-16T13:36:13.501 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.160226+0000 mon.smithi151 (mon.0) 763 : audit [DBG] from='mgr.? 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/crt"}]: dispatch 2023-11-16T13:36:13.501 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.161354+0000 mon.smithi151 (mon.0) 764 : audit [DBG] from='mgr.? 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-16T13:36:13.501 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:13.165308+0000 mon.smithi151 (mon.0) 765 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-11-16T13:36:13.501 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.165468+0000 mon.smithi151 (mon.0) 766 : audit [DBG] from='mgr.? 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/key"}]: dispatch 2023-11-16T13:36:13.501 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.166156+0000 mon.smithi151 (mon.0) 767 : audit [DBG] from='mgr.? 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:36:13.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 79071 24.06 MB/sec warmup 92 sec latency 27.572 ms 2023-11-16T13:36:14.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:13.225480+0000 mon.smithi151 (mon.0) 768 : cluster [DBG] mgrmap e22: smithi167.xxnzsz(active, starting, since 0.0669468s), standbys: smithi151.fcujra 2023-11-16T13:36:14.553 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.231843+0000 mon.smithi151 (mon.0) 769 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:36:14.553 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.232199+0000 mon.smithi151 (mon.0) 770 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:36:14.553 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.232339+0000 mon.smithi151 (mon.0) 771 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi151.mdkcfw"}]: dispatch 2023-11-16T13:36:14.553 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.232595+0000 mon.smithi151 (mon.0) 772 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:36:14.553 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.232935+0000 mon.smithi151 (mon.0) 773 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mgr metadata", "who": "smithi167.xxnzsz", "id": "smithi167.xxnzsz"}]: dispatch 2023-11-16T13:36:14.553 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.233173+0000 mon.smithi151 (mon.0) 774 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mgr metadata", "who": "smithi151.fcujra", "id": "smithi151.fcujra"}]: dispatch 2023-11-16T13:36:14.553 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.233408+0000 mon.smithi151 (mon.0) 775 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:36:14.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.233696+0000 mon.smithi151 (mon.0) 776 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:36:14.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.234008+0000 mon.smithi151 (mon.0) 777 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:36:14.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.234278+0000 mon.smithi151 (mon.0) 778 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:36:14.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.234522+0000 mon.smithi151 (mon.0) 779 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:36:14.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.234773+0000 mon.smithi151 (mon.0) 780 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:36:14.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.235032+0000 mon.smithi151 (mon.0) 781 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:36:14.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.235292+0000 mon.smithi151 (mon.0) 782 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:36:14.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.239932+0000 mon.smithi151 (mon.0) 783 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-16T13:36:14.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.240214+0000 mon.smithi151 (mon.0) 784 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-16T13:36:14.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.240848+0000 mon.smithi151 (mon.0) 785 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-16T13:36:14.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:13.329032+0000 mon.smithi151 (mon.0) 786 : cluster [INF] Manager daemon smithi167.xxnzsz is now available 2023-11-16T13:36:14.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.350342+0000 mon.smithi151 (mon.0) 787 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:14.555 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:13.350581+0000 mgr.smithi167.xxnzsz (mgr.14656) 1 : cephadm [INF] Queued nfs.foo for migration 2023-11-16T13:36:14.555 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.355301+0000 mon.smithi151 (mon.0) 788 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:36:14.555 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.356811+0000 mon.smithi151 (mon.0) 789 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:36:14.555 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.363128+0000 mon.smithi151 (mon.0) 790 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr"}]: dispatch 2023-11-16T13:36:14.555 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.378038+0000 mon.smithi151 (mon.0) 791 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:14.555 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.380529+0000 mon.smithi151 (mon.0) 792 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:36:14.555 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.399242+0000 mon.smithi151 (mon.0) 793 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi167.xxnzsz/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:36:14.555 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.409165+0000 mon.smithi151 (mon.0) 794 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi167.xxnzsz/trash_purge_schedule"}]: dispatch 2023-11-16T13:36:14.555 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:13.415417+0000 mon.smithi151 (mon.0) 795 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:14.555 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:13.783751+0000 mgr.smithi167.xxnzsz (mgr.14656) 2 : cephadm [INF] Deploying cephadm binary to smithi167 2023-11-16T13:36:14.555 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:13.829548+0000 mgr.smithi167.xxnzsz (mgr.14656) 3 : cephadm [INF] Deploying cephadm binary to smithi151 2023-11-16T13:36:14.555 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:14.122844+0000 mon.smithi151 (mon.0) 796 : cluster [DBG] Standby manager daemon smithi151.fcujra restarted 2023-11-16T13:36:14.556 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:14.122943+0000 mon.smithi151 (mon.0) 797 : cluster [DBG] Standby manager daemon smithi151.fcujra started 2023-11-16T13:36:14.556 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:14.125382+0000 mon.smithi151 (mon.0) 798 : audit [DBG] from='mgr.? 172.21.15.151:0/773057186' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi151.fcujra/crt"}]: dispatch 2023-11-16T13:36:14.556 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:14.126031+0000 mon.smithi151 (mon.0) 799 : audit [DBG] from='mgr.? 172.21.15.151:0/773057186' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-16T13:36:14.556 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:14.132112+0000 mon.smithi151 (mon.0) 800 : audit [DBG] from='mgr.? 172.21.15.151:0/773057186' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi151.fcujra/key"}]: dispatch 2023-11-16T13:36:14.556 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:14.132707+0000 mon.smithi151 (mon.0) 801 : audit [DBG] from='mgr.? 172.21.15.151:0/773057186' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:36:14.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:13.225480+0000 mon.smithi151 (mon.0) 768 : cluster [DBG] mgrmap e22: smithi167.xxnzsz(active, starting, since 0.0669468s), standbys: smithi151.fcujra 2023-11-16T13:36:14.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.231843+0000 mon.smithi151 (mon.0) 769 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:36:14.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.232199+0000 mon.smithi151 (mon.0) 770 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:36:14.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.232339+0000 mon.smithi151 (mon.0) 771 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi151.mdkcfw"}]: dispatch 2023-11-16T13:36:14.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.232595+0000 mon.smithi151 (mon.0) 772 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:36:14.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.232935+0000 mon.smithi151 (mon.0) 773 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mgr metadata", "who": "smithi167.xxnzsz", "id": "smithi167.xxnzsz"}]: dispatch 2023-11-16T13:36:14.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.233173+0000 mon.smithi151 (mon.0) 774 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mgr metadata", "who": "smithi151.fcujra", "id": "smithi151.fcujra"}]: dispatch 2023-11-16T13:36:14.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.233408+0000 mon.smithi151 (mon.0) 775 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:36:14.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.233696+0000 mon.smithi151 (mon.0) 776 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:36:14.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.234008+0000 mon.smithi151 (mon.0) 777 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:36:14.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.234278+0000 mon.smithi151 (mon.0) 778 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:36:14.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.234522+0000 mon.smithi151 (mon.0) 779 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:36:14.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.234773+0000 mon.smithi151 (mon.0) 780 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:36:14.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.235032+0000 mon.smithi151 (mon.0) 781 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:36:14.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.235292+0000 mon.smithi151 (mon.0) 782 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:36:14.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.239932+0000 mon.smithi151 (mon.0) 783 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-16T13:36:14.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.240214+0000 mon.smithi151 (mon.0) 784 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-16T13:36:14.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.240848+0000 mon.smithi151 (mon.0) 785 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-16T13:36:14.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:13.329032+0000 mon.smithi151 (mon.0) 786 : cluster [INF] Manager daemon smithi167.xxnzsz is now available 2023-11-16T13:36:14.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.350342+0000 mon.smithi151 (mon.0) 787 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:14.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:13.350581+0000 mgr.smithi167.xxnzsz (mgr.14656) 1 : cephadm [INF] Queued nfs.foo for migration 2023-11-16T13:36:14.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.355301+0000 mon.smithi151 (mon.0) 788 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:36:14.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.356811+0000 mon.smithi151 (mon.0) 789 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:36:14.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.363128+0000 mon.smithi151 (mon.0) 790 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr"}]: dispatch 2023-11-16T13:36:14.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.378038+0000 mon.smithi151 (mon.0) 791 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:14.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.380529+0000 mon.smithi151 (mon.0) 792 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:36:14.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.399242+0000 mon.smithi151 (mon.0) 793 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi167.xxnzsz/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:36:14.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.409165+0000 mon.smithi151 (mon.0) 794 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi167.xxnzsz/trash_purge_schedule"}]: dispatch 2023-11-16T13:36:14.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:13.415417+0000 mon.smithi151 (mon.0) 795 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:14.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:13.783751+0000 mgr.smithi167.xxnzsz (mgr.14656) 2 : cephadm [INF] Deploying cephadm binary to smithi167 2023-11-16T13:36:14.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:13.829548+0000 mgr.smithi167.xxnzsz (mgr.14656) 3 : cephadm [INF] Deploying cephadm binary to smithi151 2023-11-16T13:36:14.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:14.122844+0000 mon.smithi151 (mon.0) 796 : cluster [DBG] Standby manager daemon smithi151.fcujra restarted 2023-11-16T13:36:14.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:14.122943+0000 mon.smithi151 (mon.0) 797 : cluster [DBG] Standby manager daemon smithi151.fcujra started 2023-11-16T13:36:14.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:14.125382+0000 mon.smithi151 (mon.0) 798 : audit [DBG] from='mgr.? 172.21.15.151:0/773057186' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi151.fcujra/crt"}]: dispatch 2023-11-16T13:36:14.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:14.126031+0000 mon.smithi151 (mon.0) 799 : audit [DBG] from='mgr.? 172.21.15.151:0/773057186' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-16T13:36:14.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:14.132112+0000 mon.smithi151 (mon.0) 800 : audit [DBG] from='mgr.? 172.21.15.151:0/773057186' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi151.fcujra/key"}]: dispatch 2023-11-16T13:36:14.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:14.132707+0000 mon.smithi151 (mon.0) 801 : audit [DBG] from='mgr.? 172.21.15.151:0/773057186' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:36:14.638 INFO:teuthology.orchestra.run.smithi151.stdout: 5 79595 23.99 MB/sec warmup 93 sec latency 36.535 ms 2023-11-16T13:36:15.537 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:14.249785+0000 mon.smithi151 (mon.0) 802 : cluster [DBG] mgrmap e23: smithi167.xxnzsz(active, since 1.09126s), standbys: smithi151.fcujra 2023-11-16T13:36:15.537 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:14.257540+0000 mgr.smithi167.xxnzsz (mgr.14656) 4 : cluster [DBG] pgmap v3: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:36:15.537 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:14.352139+0000 mon.smithi151 (mon.0) 803 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr"}]': finished 2023-11-16T13:36:15.537 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:14.352212+0000 mon.smithi151 (mon.0) 804 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-11-16T13:36:15.537 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:14.356387+0000 mon.smithi151 (mon.0) 805 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-11-16T13:36:15.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:14.249785+0000 mon.smithi151 (mon.0) 802 : cluster [DBG] mgrmap e23: smithi167.xxnzsz(active, since 1.09126s), standbys: smithi151.fcujra 2023-11-16T13:36:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:14.257540+0000 mgr.smithi167.xxnzsz (mgr.14656) 4 : cluster [DBG] pgmap v3: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:36:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:14.352139+0000 mon.smithi151 (mon.0) 803 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr"}]': finished 2023-11-16T13:36:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:14.352212+0000 mon.smithi151 (mon.0) 804 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-11-16T13:36:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:14.356387+0000 mon.smithi151 (mon.0) 805 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-11-16T13:36:15.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 80221 23.81 MB/sec warmup 94 sec latency 49.742 ms 2023-11-16T13:36:16.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 80586 23.67 MB/sec warmup 95 sec latency 37.983 ms 2023-11-16T13:36:16.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:15.237978+0000 mgr.smithi167.xxnzsz (mgr.14656) 5 : cluster [DBG] pgmap v5: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:36:16.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:15.354518+0000 mon.smithi151 (mon.0) 806 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-11-16T13:36:16.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:15.354560+0000 mon.smithi151 (mon.0) 807 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-11-16T13:36:16.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:15.369485+0000 mon.smithi151 (mon.0) 808 : cluster [DBG] mgrmap e24: smithi167.xxnzsz(active, since 2s), standbys: smithi151.fcujra 2023-11-16T13:36:16.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:15.684179+0000 mgr.smithi167.xxnzsz (mgr.14656) 6 : cephadm [INF] [16/Nov/2023:13:36:15] ENGINE Bus STARTING 2023-11-16T13:36:16.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:15.800050+0000 mgr.smithi167.xxnzsz (mgr.14656) 7 : cephadm [INF] [16/Nov/2023:13:36:15] ENGINE Serving on https://172.21.15.167:7150 2023-11-16T13:36:16.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:15.800242+0000 mgr.smithi167.xxnzsz (mgr.14656) 8 : cephadm [INF] [16/Nov/2023:13:36:15] ENGINE Bus STARTED 2023-11-16T13:36:16.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:15.805145+0000 mon.smithi151 (mon.0) 809 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:16.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:16.181650+0000 mon.smithi151 (mon.0) 810 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:16.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:15.237978+0000 mgr.smithi167.xxnzsz (mgr.14656) 5 : cluster [DBG] pgmap v5: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:36:16.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:15.354518+0000 mon.smithi151 (mon.0) 806 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-11-16T13:36:16.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:15.354560+0000 mon.smithi151 (mon.0) 807 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-11-16T13:36:16.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:15.369485+0000 mon.smithi151 (mon.0) 808 : cluster [DBG] mgrmap e24: smithi167.xxnzsz(active, since 2s), standbys: smithi151.fcujra 2023-11-16T13:36:16.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:15.684179+0000 mgr.smithi167.xxnzsz (mgr.14656) 6 : cephadm [INF] [16/Nov/2023:13:36:15] ENGINE Bus STARTING 2023-11-16T13:36:16.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:15.800050+0000 mgr.smithi167.xxnzsz (mgr.14656) 7 : cephadm [INF] [16/Nov/2023:13:36:15] ENGINE Serving on https://172.21.15.167:7150 2023-11-16T13:36:16.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:15.800242+0000 mgr.smithi167.xxnzsz (mgr.14656) 8 : cephadm [INF] [16/Nov/2023:13:36:15] ENGINE Bus STARTED 2023-11-16T13:36:16.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:15.805145+0000 mon.smithi151 (mon.0) 809 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:16.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:16.181650+0000 mon.smithi151 (mon.0) 810 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:17.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 81458 23.78 MB/sec warmup 96 sec latency 29.204 ms 2023-11-16T13:36:18.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:17.238336+0000 mgr.smithi167.xxnzsz (mgr.14656) 9 : cluster [DBG] pgmap v7: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:36:18.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:17.368053+0000 mon.smithi151 (mon.0) 811 : cluster [DBG] mgrmap e25: smithi167.xxnzsz(active, since 4s), standbys: smithi151.fcujra 2023-11-16T13:36:18.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 81986 23.75 MB/sec warmup 97 sec latency 26.439 ms 2023-11-16T13:36:18.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:17.238336+0000 mgr.smithi167.xxnzsz (mgr.14656) 9 : cluster [DBG] pgmap v7: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:36:18.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:17.368053+0000 mon.smithi151 (mon.0) 811 : cluster [DBG] mgrmap e25: smithi167.xxnzsz(active, since 4s), standbys: smithi151.fcujra 2023-11-16T13:36:19.637 INFO:teuthology.orchestra.run.smithi151.stdout: 5 82534 23.56 MB/sec warmup 98 sec latency 47.041 ms 2023-11-16T13:36:19.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:18.467710+0000 mon.smithi151 (mon.0) 812 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:19.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:18.473681+0000 mon.smithi151 (mon.0) 813 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:19.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:18.751664+0000 mon.smithi151 (mon.0) 814 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:19.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:18.467710+0000 mon.smithi151 (mon.0) 812 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:19.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:18.473681+0000 mon.smithi151 (mon.0) 813 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:19.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:18.751664+0000 mon.smithi151 (mon.0) 814 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:20.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 83233 23.54 MB/sec warmup 99 sec latency 27.224 ms 2023-11-16T13:36:20.770 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:20 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:19.238810+0000 mgr.smithi167.xxnzsz (mgr.14656) 10 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:36:20.770 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:20 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:19.517698+0000 mon.smithi151 (mon.0) 815 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:20.770 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:20 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:19.523646+0000 mon.smithi151 (mon.0) 816 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:20.770 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:20 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:19.579572+0000 mon.smithi151 (mon.0) 817 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:20.770 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:20 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:19.586722+0000 mon.smithi151 (mon.0) 818 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:20.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:20 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:19.238810+0000 mgr.smithi167.xxnzsz (mgr.14656) 10 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:36:20.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:20 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:19.517698+0000 mon.smithi151 (mon.0) 815 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:20.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:20 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:19.523646+0000 mon.smithi151 (mon.0) 816 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:20.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:20 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:19.579572+0000 mon.smithi151 (mon.0) 817 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:20.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:20 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:19.586722+0000 mon.smithi151 (mon.0) 818 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:21.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 84017 23.39 MB/sec warmup 100 sec latency 195.377 ms 2023-11-16T13:36:22.267 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:20.581407+0000 mon.smithi151 (mon.0) 819 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:22.267 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:20.588033+0000 mon.smithi151 (mon.0) 820 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:22.267 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:20.665378+0000 mon.smithi151 (mon.0) 821 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-11-16T13:36:22.267 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:21.281391+0000 mon.smithi151 (mon.0) 822 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-11-16T13:36:22.267 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:21.282806+0000 mon.smithi151 (mon.0) 823 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:36:22.267 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:21.283112+0000 mon.smithi151 (mon.0) 824 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:36:22.267 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:21.289024+0000 mon.smithi151 (mon.0) 825 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:36:22.267 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:21.289252+0000 mon.smithi151 (mon.0) 826 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:36:22.268 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:20.581407+0000 mon.smithi151 (mon.0) 819 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:22.268 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:20.588033+0000 mon.smithi151 (mon.0) 820 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:22.268 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:20.665378+0000 mon.smithi151 (mon.0) 821 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-11-16T13:36:22.268 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:21.281391+0000 mon.smithi151 (mon.0) 822 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-11-16T13:36:22.268 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:21.282806+0000 mon.smithi151 (mon.0) 823 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:36:22.268 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:21.283112+0000 mon.smithi151 (mon.0) 824 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:36:22.268 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:21.289024+0000 mon.smithi151 (mon.0) 825 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:36:22.268 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:21 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:21.289252+0000 mon.smithi151 (mon.0) 826 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:36:22.637 INFO:teuthology.orchestra.run.smithi151.stdout: 5 84495 23.37 MB/sec warmup 101 sec latency 28.849 ms 2023-11-16T13:36:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:21.239434+0000 mgr.smithi167.xxnzsz (mgr.14656) 11 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 6.4 MiB/s wr, 620 op/s 2023-11-16T13:36:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:21.288461+0000 mon.smithi167 (mon.1) 17 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-11-16T13:36:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:21.846211+0000 mon.smithi167 (mon.1) 18 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-11-16T13:36:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:22.019185+0000 mon.smithi151 (mon.0) 827 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:22.024197+0000 mon.smithi151 (mon.0) 828 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:22.025388+0000 mon.smithi151 (mon.0) 829 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:22.026074+0000 mon.smithi151 (mon.0) 830 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:22.026824+0000 mon.smithi151 (mon.0) 831 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:22.027416+0000 mon.smithi151 (mon.0) 832 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:23.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:22.031686+0000 mon.smithi151 (mon.0) 833 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:21.239434+0000 mgr.smithi167.xxnzsz (mgr.14656) 11 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 6.4 MiB/s wr, 620 op/s 2023-11-16T13:36:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:21.288461+0000 mon.smithi167 (mon.1) 17 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-11-16T13:36:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:21.846211+0000 mon.smithi167 (mon.1) 18 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-11-16T13:36:23.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:22.019185+0000 mon.smithi151 (mon.0) 827 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:23.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:22.024197+0000 mon.smithi151 (mon.0) 828 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:23.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:22.025388+0000 mon.smithi151 (mon.0) 829 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:23.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:22.026074+0000 mon.smithi151 (mon.0) 830 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:23.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:22.026824+0000 mon.smithi151 (mon.0) 831 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:23.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:22.027416+0000 mon.smithi151 (mon.0) 832 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:23.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:22.031686+0000 mon.smithi151 (mon.0) 833 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 85023 23.32 MB/sec warmup 102 sec latency 34.376 ms 2023-11-16T13:36:24.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:22.027821+0000 mgr.smithi167.xxnzsz (mgr.14656) 12 : cephadm [INF] Adjusting osd_memory_target on smithi167 to 2712M 2023-11-16T13:36:24.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:23.662929+0000 mon.smithi151 (mon.0) 834 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:24.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:23.667870+0000 mon.smithi151 (mon.0) 835 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:24.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:23.669120+0000 mon.smithi151 (mon.0) 836 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:24.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:23.669856+0000 mon.smithi151 (mon.0) 837 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:24.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:23.670495+0000 mon.smithi151 (mon.0) 838 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:24.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:23.671139+0000 mon.smithi151 (mon.0) 839 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:24.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:23.676693+0000 mon.smithi151 (mon.0) 840 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:24.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:23.677754+0000 mon.smithi151 (mon.0) 841 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:24.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:23 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:23.678459+0000 mon.smithi151 (mon.0) 842 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:36:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:22.027821+0000 mgr.smithi167.xxnzsz (mgr.14656) 12 : cephadm [INF] Adjusting osd_memory_target on smithi167 to 2712M 2023-11-16T13:36:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:23.662929+0000 mon.smithi151 (mon.0) 834 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:23.667870+0000 mon.smithi151 (mon.0) 835 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:24.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:23.669120+0000 mon.smithi151 (mon.0) 836 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:23.669856+0000 mon.smithi151 (mon.0) 837 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:23.670495+0000 mon.smithi151 (mon.0) 838 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:23.671139+0000 mon.smithi151 (mon.0) 839 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-11-16T13:36:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:23.676693+0000 mon.smithi151 (mon.0) 840 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:23.677754+0000 mon.smithi151 (mon.0) 841 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:24.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:23.678459+0000 mon.smithi151 (mon.0) 842 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:36:24.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 85352 23.26 MB/sec warmup 103 sec latency 49.104 ms 2023-11-16T13:36:25.233 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:23.239781+0000 mgr.smithi167.xxnzsz (mgr.14656) 13 : cluster [DBG] pgmap v10: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 5.1 MiB/s wr, 487 op/s 2023-11-16T13:36:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:23.671773+0000 mgr.smithi167.xxnzsz (mgr.14656) 14 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 1944M 2023-11-16T13:36:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:23.679565+0000 mgr.smithi167.xxnzsz (mgr.14656) 15 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-11-16T13:36:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:23.679724+0000 mgr.smithi167.xxnzsz (mgr.14656) 16 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.conf 2023-11-16T13:36:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:23.945595+0000 mgr.smithi167.xxnzsz (mgr.14656) 17 : cephadm [INF] Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:36:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:23.994521+0000 mgr.smithi167.xxnzsz (mgr.14656) 18 : cephadm [INF] Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:36:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:24.749130+0000 mon.smithi151 (mon.0) 843 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:25.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:24.754921+0000 mon.smithi151 (mon.0) 844 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:23.239781+0000 mgr.smithi167.xxnzsz (mgr.14656) 13 : cluster [DBG] pgmap v10: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 5.1 MiB/s wr, 487 op/s 2023-11-16T13:36:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:23.671773+0000 mgr.smithi167.xxnzsz (mgr.14656) 14 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 1944M 2023-11-16T13:36:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:23.679565+0000 mgr.smithi167.xxnzsz (mgr.14656) 15 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-11-16T13:36:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:23.679724+0000 mgr.smithi167.xxnzsz (mgr.14656) 16 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.conf 2023-11-16T13:36:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:23.945595+0000 mgr.smithi167.xxnzsz (mgr.14656) 17 : cephadm [INF] Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:36:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:23.994521+0000 mgr.smithi167.xxnzsz (mgr.14656) 18 : cephadm [INF] Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:36:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:24.749130+0000 mon.smithi151 (mon.0) 843 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:25.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:24.754921+0000 mon.smithi151 (mon.0) 844 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:25.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 85647 23.07 MB/sec warmup 104 sec latency 242.545 ms 2023-11-16T13:36:26.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:24.211188+0000 mgr.smithi167.xxnzsz (mgr.14656) 19 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:36:26.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:24.312954+0000 mgr.smithi167.xxnzsz (mgr.14656) 20 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:36:26.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:24.478894+0000 mgr.smithi167.xxnzsz (mgr.14656) 21 : cephadm [INF] Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:36:26.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:24.680187+0000 mgr.smithi167.xxnzsz (mgr.14656) 22 : cephadm [INF] Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:36:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:25.001062+0000 mon.smithi151 (mon.0) 845 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:25.005716+0000 mon.smithi151 (mon.0) 846 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:25.009857+0000 mon.smithi151 (mon.0) 847 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:25.034299+0000 mon.smithi151 (mon.0) 848 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2023-11-16T13:36:26.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:24.211188+0000 mgr.smithi167.xxnzsz (mgr.14656) 19 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:36:26.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:24.312954+0000 mgr.smithi167.xxnzsz (mgr.14656) 20 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:36:26.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:24.478894+0000 mgr.smithi167.xxnzsz (mgr.14656) 21 : cephadm [INF] Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:36:26.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:24.680187+0000 mgr.smithi167.xxnzsz (mgr.14656) 22 : cephadm [INF] Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:36:26.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:25.001062+0000 mon.smithi151 (mon.0) 845 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:26.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:25.005716+0000 mon.smithi151 (mon.0) 846 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:26.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:25.009857+0000 mon.smithi151 (mon.0) 847 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:26.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:25.034299+0000 mon.smithi151 (mon.0) 848 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2023-11-16T13:36:26.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 86186 22.93 MB/sec warmup 105 sec latency 27.411 ms 2023-11-16T13:36:27.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:25.240301+0000 mgr.smithi167.xxnzsz (mgr.14656) 23 : cluster [DBG] pgmap v11: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 8.6 MiB/s wr, 832 op/s 2023-11-16T13:36:27.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:26.018234+0000 mon.smithi151 (mon.0) 849 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2023-11-16T13:36:27.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:26.018306+0000 mon.smithi151 (mon.0) 850 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-11-16T13:36:27.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:26.023352+0000 mon.smithi151 (mon.0) 851 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2023-11-16T13:36:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:25.240301+0000 mgr.smithi167.xxnzsz (mgr.14656) 23 : cluster [DBG] pgmap v11: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 8.6 MiB/s wr, 832 op/s 2023-11-16T13:36:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:26.018234+0000 mon.smithi151 (mon.0) 849 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2023-11-16T13:36:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:26.018306+0000 mon.smithi151 (mon.0) 850 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-11-16T13:36:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:27 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:26.023352+0000 mon.smithi151 (mon.0) 851 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2023-11-16T13:36:27.637 INFO:teuthology.orchestra.run.smithi151.stdout: 5 86732 22.86 MB/sec warmup 106 sec latency 65.396 ms 2023-11-16T13:36:28.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:27.018366+0000 mon.smithi151 (mon.0) 852 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2023-11-16T13:36:28.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:27.018486+0000 mon.smithi151 (mon.0) 853 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-11-16T13:36:28.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:27.020406+0000 mgr.smithi167.xxnzsz (mgr.14656) 24 : cephadm [INF] Migrating nfs.foo from legacy pool nfs-ganesha namespace foo 2023-11-16T13:36:28.331 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:27.018366+0000 mon.smithi151 (mon.0) 852 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2023-11-16T13:36:28.331 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:27.018486+0000 mon.smithi151 (mon.0) 853 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-11-16T13:36:28.331 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:27.020406+0000 mgr.smithi167.xxnzsz (mgr.14656) 24 : cephadm [INF] Migrating nfs.foo from legacy pool nfs-ganesha namespace foo 2023-11-16T13:36:28.636 INFO:teuthology.orchestra.run.smithi151.stdout: 5 86732 22.64 MB/sec warmup 107 sec latency 1065.467 ms 2023-11-16T13:36:29.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:27.056691+0000 mgr.smithi167.xxnzsz (mgr.14656) 25 : cephadm [INF] Found 1 exports for legacy nfs.foo 2023-11-16T13:36:29.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:27.056797+0000 mgr.smithi167.xxnzsz (mgr.14656) 26 : cephadm [INF] Removing old nfs.foo daemons ['nfs.foo.0.0.smithi151.cywuiq', 'nfs.foo.1.0.smithi167.pnnuhc'] 2023-11-16T13:36:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:27.056882+0000 mgr.smithi167.xxnzsz (mgr.14656) 27 : cephadm [INF] Remove daemons nfs.foo.0.0.smithi151.cywuiq nfs.foo.1.0.smithi167.pnnuhc 2023-11-16T13:36:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:27.057262+0000 mgr.smithi167.xxnzsz (mgr.14656) 28 : cephadm [INF] Removing daemon nfs.foo.0.0.smithi151.cywuiq from smithi151 -- ports [2049] 2023-11-16T13:36:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:27.057514+0000 mgr.smithi167.xxnzsz (mgr.14656) 29 : cephadm [INF] Removing daemon nfs.foo.1.0.smithi167.pnnuhc from smithi167 -- ports [2049] 2023-11-16T13:36:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:27.240659+0000 mgr.smithi167.xxnzsz (mgr.14656) 30 : cluster [DBG] pgmap v14: 129 pgs: 32 unknown, 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2023-11-16T13:36:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:28.022703+0000 mgr.smithi167.xxnzsz (mgr.14656) 31 : cephadm [INF] Removing key for client.nfs.foo.0.0.smithi151.cywuiq 2023-11-16T13:36:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:28.023899+0000 mon.smithi151 (mon.0) 854 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi151.cywuiq"}]: dispatch 2023-11-16T13:36:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:28.037900+0000 mon.smithi151 (mon.0) 855 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-11-16T13:36:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:28.038241+0000 mon.smithi151 (mon.0) 856 : cluster [DBG] mgrmap e26: smithi167.xxnzsz(active, since 14s), standbys: smithi151.fcujra 2023-11-16T13:36:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:28.038284+0000 mon.smithi151 (mon.0) 857 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi151.cywuiq"}]': finished 2023-11-16T13:36:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:28.039193+0000 mon.smithi151 (mon.0) 858 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi151.cywuiq-rgw"}]: dispatch 2023-11-16T13:36:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:28.042176+0000 mon.smithi151 (mon.0) 859 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi151.cywuiq-rgw"}]': finished 2023-11-16T13:36:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:28.380895+0000 mon.smithi151 (mon.0) 860 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:36:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:27.056691+0000 mgr.smithi167.xxnzsz (mgr.14656) 25 : cephadm [INF] Found 1 exports for legacy nfs.foo 2023-11-16T13:36:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:27.056797+0000 mgr.smithi167.xxnzsz (mgr.14656) 26 : cephadm [INF] Removing old nfs.foo daemons ['nfs.foo.0.0.smithi151.cywuiq', 'nfs.foo.1.0.smithi167.pnnuhc'] 2023-11-16T13:36:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:27.056882+0000 mgr.smithi167.xxnzsz (mgr.14656) 27 : cephadm [INF] Remove daemons nfs.foo.0.0.smithi151.cywuiq nfs.foo.1.0.smithi167.pnnuhc 2023-11-16T13:36:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:27.057262+0000 mgr.smithi167.xxnzsz (mgr.14656) 28 : cephadm [INF] Removing daemon nfs.foo.0.0.smithi151.cywuiq from smithi151 -- ports [2049] 2023-11-16T13:36:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:27.057514+0000 mgr.smithi167.xxnzsz (mgr.14656) 29 : cephadm [INF] Removing daemon nfs.foo.1.0.smithi167.pnnuhc from smithi167 -- ports [2049] 2023-11-16T13:36:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:27.240659+0000 mgr.smithi167.xxnzsz (mgr.14656) 30 : cluster [DBG] pgmap v14: 129 pgs: 32 unknown, 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2023-11-16T13:36:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:28.022703+0000 mgr.smithi167.xxnzsz (mgr.14656) 31 : cephadm [INF] Removing key for client.nfs.foo.0.0.smithi151.cywuiq 2023-11-16T13:36:29.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:28.023899+0000 mon.smithi151 (mon.0) 854 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi151.cywuiq"}]: dispatch 2023-11-16T13:36:29.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:28.037900+0000 mon.smithi151 (mon.0) 855 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-11-16T13:36:29.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:28.038241+0000 mon.smithi151 (mon.0) 856 : cluster [DBG] mgrmap e26: smithi167.xxnzsz(active, since 14s), standbys: smithi151.fcujra 2023-11-16T13:36:29.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:28.038284+0000 mon.smithi151 (mon.0) 857 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi151.cywuiq"}]': finished 2023-11-16T13:36:29.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:28.039193+0000 mon.smithi151 (mon.0) 858 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi151.cywuiq-rgw"}]: dispatch 2023-11-16T13:36:29.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:28.042176+0000 mon.smithi151 (mon.0) 859 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi151.cywuiq-rgw"}]': finished 2023-11-16T13:36:29.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:28.380895+0000 mon.smithi151 (mon.0) 860 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:36:29.637 INFO:teuthology.orchestra.run.smithi151.stdout: 5 86732 22.43 MB/sec warmup 108 sec latency 2065.532 ms 2023-11-16T13:36:30.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:30 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:28.038903+0000 mgr.smithi167.xxnzsz (mgr.14656) 32 : cephadm [INF] Removing key for client.nfs.foo.0.0.smithi151.cywuiq-rgw 2023-11-16T13:36:30.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:30 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:28.038903+0000 mgr.smithi167.xxnzsz (mgr.14656) 32 : cephadm [INF] Removing key for client.nfs.foo.0.0.smithi151.cywuiq-rgw 2023-11-16T13:36:30.589 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:36:30.637 INFO:teuthology.orchestra.run.smithi151.stdout: 5 86732 22.23 MB/sec warmup 109 sec latency 3065.589 ms 2023-11-16T13:36:31.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:29.241198+0000 mgr.smithi167.xxnzsz (mgr.14656) 33 : cluster [DBG] pgmap v16: 129 pgs: 32 unknown, 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.9 MiB/s wr, 664 op/s 2023-11-16T13:36:31.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.574178+0000 mgr.smithi167.xxnzsz (mgr.14656) 34 : audit [DBG] from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:31.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:30.729503+0000 mgr.smithi167.xxnzsz (mgr.14656) 35 : cephadm [INF] Removing key for client.nfs.foo.1.0.smithi167.pnnuhc 2023-11-16T13:36:31.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.729907+0000 mon.smithi151 (mon.0) 861 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc"}]: dispatch 2023-11-16T13:36:31.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.733369+0000 mon.smithi151 (mon.0) 862 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc"}]': finished 2023-11-16T13:36:31.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:30.733795+0000 mgr.smithi167.xxnzsz (mgr.14656) 36 : cephadm [INF] Removing key for client.nfs.foo.1.0.smithi167.pnnuhc-rgw 2023-11-16T13:36:31.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.734049+0000 mon.smithi151 (mon.0) 863 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc-rgw"}]: dispatch 2023-11-16T13:36:31.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.739059+0000 mon.smithi151 (mon.0) 864 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc-rgw"}]': finished 2023-11-16T13:36:31.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.744417+0000 mon.smithi151 (mon.0) 865 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.744814+0000 mon.smithi151 (mon.0) 866 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "nfs export apply", "cluster_id": "foo"}]: dispatch 2023-11-16T13:36:31.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.745315+0000 mgr.smithi167.xxnzsz (mgr.14656) 37 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "nfs export apply", "cluster_id": "foo"}]: dispatch 2023-11-16T13:36:31.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.767059+0000 mon.smithi151 (mon.0) 867 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2023-11-16T13:36:31.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.772315+0000 mon.smithi151 (mon.0) 868 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2023-11-16T13:36:31.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:30.776278+0000 mgr.smithi167.xxnzsz (mgr.14656) 38 : cephadm [INF] Done migrating nfs.foo 2023-11-16T13:36:31.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:30.776338+0000 mgr.smithi167.xxnzsz (mgr.14656) 39 : cephadm [INF] Done migrating all NFS services 2023-11-16T13:36:31.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.779529+0000 mon.smithi151 (mon.0) 869 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.784350+0000 mon.smithi151 (mon.0) 870 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.792912+0000 mon.smithi151 (mon.0) 871 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.804814+0000 mon.smithi151 (mon.0) 872 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:30.805131+0000 mgr.smithi167.xxnzsz (mgr.14656) 40 : cephadm [INF] Fencing old nfs.foo.0.0.smithi151.cywuiq 2023-11-16T13:36:31.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.805489+0000 mon.smithi151 (mon.0) 873 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi151.cywuiq"}]: dispatch 2023-11-16T13:36:31.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.808663+0000 mon.smithi151 (mon.0) 874 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:30.808923+0000 mgr.smithi167.xxnzsz (mgr.14656) 41 : cephadm [INF] Fencing old nfs.foo.1.0.smithi167.pnnuhc 2023-11-16T13:36:31.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.809204+0000 mon.smithi151 (mon.0) 875 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc"}]: dispatch 2023-11-16T13:36:31.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.812141+0000 mon.smithi151 (mon.0) 876 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:30.812511+0000 mgr.smithi167.xxnzsz (mgr.14656) 42 : cephadm [INF] Creating key for client.nfs.foo.0.1.smithi151.qfkjic 2023-11-16T13:36:31.100 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.812799+0000 mon.smithi151 (mon.0) 877 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi151.qfkjic", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2023-11-16T13:36:31.100 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.815468+0000 mon.smithi151 (mon.0) 878 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi151.qfkjic", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2023-11-16T13:36:31.100 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:30.815859+0000 mgr.smithi167.xxnzsz (mgr.14656) 43 : cephadm [INF] Ensuring nfs.foo.0 is in the ganesha grace table 2023-11-16T13:36:31.100 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.816094+0000 mon.smithi151 (mon.0) 879 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2023-11-16T13:36:31.100 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.818726+0000 mon.smithi151 (mon.0) 880 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2023-11-16T13:36:31.100 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.819697+0000 mon.smithi151 (mon.0) 881 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:31.100 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.861096+0000 mon.smithi151 (mon.0) 882 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2023-11-16T13:36:31.100 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.864128+0000 mon.smithi151 (mon.0) 883 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2023-11-16T13:36:31.100 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.890453+0000 mgr.smithi167.xxnzsz (mgr.14656) 44 : audit [DBG] from='client.14698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:31.100 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:30.910406+0000 mgr.smithi167.xxnzsz (mgr.14656) 45 : cephadm [INF] Rados config object exists: conf-nfs.foo 2023-11-16T13:36:31.100 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:30.910501+0000 mgr.smithi167.xxnzsz (mgr.14656) 46 : cephadm [INF] Creating key for client.nfs.foo.0.1.smithi151.qfkjic-rgw 2023-11-16T13:36:31.101 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.910931+0000 mon.smithi151 (mon.0) 884 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi151.qfkjic-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2023-11-16T13:36:31.101 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.914320+0000 mon.smithi151 (mon.0) 885 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi151.qfkjic-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2023-11-16T13:36:31.101 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:30.915005+0000 mgr.smithi167.xxnzsz (mgr.14656) 47 : cephadm [WRN] Bind address in nfs.foo.0.1.smithi151.qfkjic's ganesha conf is defaulting to empty 2023-11-16T13:36:31.101 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:30.920277+0000 mon.smithi151 (mon.0) 886 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:31.101 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:30.921282+0000 mgr.smithi167.xxnzsz (mgr.14656) 48 : cephadm [INF] Deploying daemon nfs.foo.0.1.smithi151.qfkjic on smithi151 2023-11-16T13:36:31.224 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:36:31.224 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (4m) 11s ago 5m 23.1M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:36:31.225 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (5m) 11s ago 5m 7008k - 16.2.5 6933c2a0b7dd 954bd32d4ee2 2023-11-16T13:36:31.225 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (5m) 12s ago 5m 7155k - 16.2.5 6933c2a0b7dd 649801b49aea 2023-11-16T13:36:31.225 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (4m) 11s ago 5m 35.8M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:36:31.225 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (2m) 11s ago 2m 14.2M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:36:31.225 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (2m) 12s ago 2m 41.9M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:36:31.225 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:9283 running (6m) 11s ago 6m 388M - 16.2.5 6933c2a0b7dd a009db54a97f 2023-11-16T13:36:31.225 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (27s) 12s ago 5m 471M - 17.2.7-68-gae6ce92c ea201341e8f3 d8a5483bcd6d 2023-11-16T13:36:31.225 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (6m) 11s ago 6m 112M 2048M 16.2.5 6933c2a0b7dd 3a504a936c5d 2023-11-16T13:36:31.225 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (4m) 12s ago 4m 71.2M 2048M 16.2.5 6933c2a0b7dd 448b57d6232a 2023-11-16T13:36:31.225 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (5m) 11s ago 5m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:36:31.225 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (4m) 12s ago 4m 18.2M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:36:31.226 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (4m) 11s ago 4m 1093M 1944M 16.2.5 6933c2a0b7dd 7598578fb701 2023-11-16T13:36:31.226 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (4m) 11s ago 4m 936M 1944M 16.2.5 6933c2a0b7dd 989f26649fd3 2023-11-16T13:36:31.226 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (4m) 11s ago 4m 618M 1944M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:36:31.226 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (3m) 11s ago 3m 1046M 1944M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:36:31.226 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (3m) 12s ago 3m 967M 2712M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:36:31.226 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (3m) 12s ago 3m 1269M 2712M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:36:31.226 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (3m) 12s ago 3m 989M 2712M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:36:31.226 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (3m) 12s ago 3m 1618M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:36:31.226 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (4m) 11s ago 5m 46.5M - 2.18.1 de242295e225 336d5d209623 2023-11-16T13:36:31.329 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:29.241198+0000 mgr.smithi167.xxnzsz (mgr.14656) 33 : cluster [DBG] pgmap v16: 129 pgs: 32 unknown, 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.9 MiB/s wr, 664 op/s 2023-11-16T13:36:31.329 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.574178+0000 mgr.smithi167.xxnzsz (mgr.14656) 34 : audit [DBG] from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:31.329 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:30.729503+0000 mgr.smithi167.xxnzsz (mgr.14656) 35 : cephadm [INF] Removing key for client.nfs.foo.1.0.smithi167.pnnuhc 2023-11-16T13:36:31.329 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.729907+0000 mon.smithi151 (mon.0) 861 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc"}]: dispatch 2023-11-16T13:36:31.329 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.733369+0000 mon.smithi151 (mon.0) 862 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc"}]': finished 2023-11-16T13:36:31.329 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:30.733795+0000 mgr.smithi167.xxnzsz (mgr.14656) 36 : cephadm [INF] Removing key for client.nfs.foo.1.0.smithi167.pnnuhc-rgw 2023-11-16T13:36:31.329 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.734049+0000 mon.smithi151 (mon.0) 863 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc-rgw"}]: dispatch 2023-11-16T13:36:31.329 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.739059+0000 mon.smithi151 (mon.0) 864 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc-rgw"}]': finished 2023-11-16T13:36:31.329 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.744417+0000 mon.smithi151 (mon.0) 865 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.329 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.744814+0000 mon.smithi151 (mon.0) 866 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "nfs export apply", "cluster_id": "foo"}]: dispatch 2023-11-16T13:36:31.330 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.745315+0000 mgr.smithi167.xxnzsz (mgr.14656) 37 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "nfs export apply", "cluster_id": "foo"}]: dispatch 2023-11-16T13:36:31.330 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.767059+0000 mon.smithi151 (mon.0) 867 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2023-11-16T13:36:31.330 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.772315+0000 mon.smithi151 (mon.0) 868 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2023-11-16T13:36:31.330 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:30.776278+0000 mgr.smithi167.xxnzsz (mgr.14656) 38 : cephadm [INF] Done migrating nfs.foo 2023-11-16T13:36:31.330 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:30.776338+0000 mgr.smithi167.xxnzsz (mgr.14656) 39 : cephadm [INF] Done migrating all NFS services 2023-11-16T13:36:31.330 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.779529+0000 mon.smithi151 (mon.0) 869 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.330 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.784350+0000 mon.smithi151 (mon.0) 870 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.330 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.792912+0000 mon.smithi151 (mon.0) 871 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.330 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.804814+0000 mon.smithi151 (mon.0) 872 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.330 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:30.805131+0000 mgr.smithi167.xxnzsz (mgr.14656) 40 : cephadm [INF] Fencing old nfs.foo.0.0.smithi151.cywuiq 2023-11-16T13:36:31.330 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.805489+0000 mon.smithi151 (mon.0) 873 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi151.cywuiq"}]: dispatch 2023-11-16T13:36:31.330 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.808663+0000 mon.smithi151 (mon.0) 874 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.331 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:30.808923+0000 mgr.smithi167.xxnzsz (mgr.14656) 41 : cephadm [INF] Fencing old nfs.foo.1.0.smithi167.pnnuhc 2023-11-16T13:36:31.331 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.809204+0000 mon.smithi151 (mon.0) 875 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi167.pnnuhc"}]: dispatch 2023-11-16T13:36:31.331 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.812141+0000 mon.smithi151 (mon.0) 876 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:31.331 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:30.812511+0000 mgr.smithi167.xxnzsz (mgr.14656) 42 : cephadm [INF] Creating key for client.nfs.foo.0.1.smithi151.qfkjic 2023-11-16T13:36:31.331 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.812799+0000 mon.smithi151 (mon.0) 877 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi151.qfkjic", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2023-11-16T13:36:31.331 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.815468+0000 mon.smithi151 (mon.0) 878 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi151.qfkjic", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2023-11-16T13:36:31.331 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:30.815859+0000 mgr.smithi167.xxnzsz (mgr.14656) 43 : cephadm [INF] Ensuring nfs.foo.0 is in the ganesha grace table 2023-11-16T13:36:31.331 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.816094+0000 mon.smithi151 (mon.0) 879 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2023-11-16T13:36:31.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.818726+0000 mon.smithi151 (mon.0) 880 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2023-11-16T13:36:31.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.819697+0000 mon.smithi151 (mon.0) 881 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:31.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.861096+0000 mon.smithi151 (mon.0) 882 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2023-11-16T13:36:31.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.864128+0000 mon.smithi151 (mon.0) 883 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2023-11-16T13:36:31.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.890453+0000 mgr.smithi167.xxnzsz (mgr.14656) 44 : audit [DBG] from='client.14698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:31.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:30.910406+0000 mgr.smithi167.xxnzsz (mgr.14656) 45 : cephadm [INF] Rados config object exists: conf-nfs.foo 2023-11-16T13:36:31.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:30.910501+0000 mgr.smithi167.xxnzsz (mgr.14656) 46 : cephadm [INF] Creating key for client.nfs.foo.0.1.smithi151.qfkjic-rgw 2023-11-16T13:36:31.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.910931+0000 mon.smithi151 (mon.0) 884 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi151.qfkjic-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2023-11-16T13:36:31.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.914320+0000 mon.smithi151 (mon.0) 885 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi151.qfkjic-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2023-11-16T13:36:31.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:30.915005+0000 mgr.smithi167.xxnzsz (mgr.14656) 47 : cephadm [WRN] Bind address in nfs.foo.0.1.smithi151.qfkjic's ganesha conf is defaulting to empty 2023-11-16T13:36:31.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:30.920277+0000 mon.smithi151 (mon.0) 886 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:31.332 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:31 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:30.921282+0000 mgr.smithi167.xxnzsz (mgr.14656) 48 : cephadm [INF] Deploying daemon nfs.foo.0.1.smithi151.qfkjic on smithi151 2023-11-16T13:36:31.613 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:36:31.614 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:36:31.614 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:36:31.614 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:36:31.614 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:36:31.614 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1, 2023-11-16T13:36:31.614 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 1 2023-11-16T13:36:31.614 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:36:31.614 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:36:31.614 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2023-11-16T13:36:31.615 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:36:31.615 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:36:31.615 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:36:31.615 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:36:31.615 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:36:31.615 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 13, 2023-11-16T13:36:31.615 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 1 2023-11-16T13:36:31.615 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:36:31.615 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:36:31.637 INFO:teuthology.orchestra.run.smithi151.stdout: 5 86732 22.02 MB/sec warmup 110 sec latency 4065.638 ms 2023-11-16T13:36:31.939 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:36:31.939 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:36:31.939 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:36:31.939 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:36:31.939 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [], 2023-11-16T13:36:31.939 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "1/21 daemons upgraded", 2023-11-16T13:36:31.940 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "", 2023-11-16T13:36:31.940 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:36:31.940 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:36:32.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:31.212978+0000 mgr.smithi167.xxnzsz (mgr.14656) 49 : audit [DBG] from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:32.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:31.241890+0000 mgr.smithi167.xxnzsz (mgr.14656) 50 : cluster [DBG] pgmap v17: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 3.7 MiB/s wr, 429 op/s 2023-11-16T13:36:32.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:31.612601+0000 mon.smithi151 (mon.0) 887 : audit [DBG] from='client.? 172.21.15.151:0/1974388538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:36:32.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:31.937191+0000 mgr.smithi167.xxnzsz (mgr.14656) 51 : audit [DBG] from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:32.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:32.046637+0000 mon.smithi151 (mon.0) 888 : cluster [DBG] mgrmap e27: smithi167.xxnzsz(active, since 18s), standbys: smithi151.fcujra 2023-11-16T13:36:32.637 INFO:teuthology.orchestra.run.smithi151.stdout: 5 86732 21.83 MB/sec warmup 111 sec latency 5065.729 ms 2023-11-16T13:36:32.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:31.212978+0000 mgr.smithi167.xxnzsz (mgr.14656) 49 : audit [DBG] from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:32.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:31.241890+0000 mgr.smithi167.xxnzsz (mgr.14656) 50 : cluster [DBG] pgmap v17: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 3.7 MiB/s wr, 429 op/s 2023-11-16T13:36:32.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:31.612601+0000 mon.smithi151 (mon.0) 887 : audit [DBG] from='client.? 172.21.15.151:0/1974388538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:36:32.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:31.937191+0000 mgr.smithi167.xxnzsz (mgr.14656) 51 : audit [DBG] from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:36:32.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:32.046637+0000 mon.smithi151 (mon.0) 888 : cluster [DBG] mgrmap e27: smithi167.xxnzsz(active, since 18s), standbys: smithi151.fcujra 2023-11-16T13:36:33.637 INFO:teuthology.orchestra.run.smithi151.stdout: 5 86732 21.63 MB/sec warmup 112 sec latency 6065.814 ms 2023-11-16T13:36:33.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:32.557809+0000 mon.smithi151 (mon.0) 889 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:33.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:32.561886+0000 mon.smithi151 (mon.0) 890 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:33.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:32.591172+0000 mon.smithi151 (mon.0) 891 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:33.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:32.591595+0000 mgr.smithi167.xxnzsz (mgr.14656) 52 : cephadm [INF] Creating key for client.nfs.foo.1.1.smithi167.rpoanb 2023-11-16T13:36:33.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:32.591862+0000 mon.smithi151 (mon.0) 892 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi167.rpoanb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2023-11-16T13:36:33.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:32.594231+0000 mon.smithi151 (mon.0) 893 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi167.rpoanb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2023-11-16T13:36:33.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:32.594784+0000 mgr.smithi167.xxnzsz (mgr.14656) 53 : cephadm [INF] Ensuring nfs.foo.1 is in the ganesha grace table 2023-11-16T13:36:33.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:32.595130+0000 mon.smithi151 (mon.0) 894 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2023-11-16T13:36:33.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:32.597700+0000 mon.smithi151 (mon.0) 895 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2023-11-16T13:36:33.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:32.598797+0000 mon.smithi151 (mon.0) 896 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:33.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:32.557809+0000 mon.smithi151 (mon.0) 889 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:33.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:32.561886+0000 mon.smithi151 (mon.0) 890 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:33.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:32.591172+0000 mon.smithi151 (mon.0) 891 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:33.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:32.591595+0000 mgr.smithi167.xxnzsz (mgr.14656) 52 : cephadm [INF] Creating key for client.nfs.foo.1.1.smithi167.rpoanb 2023-11-16T13:36:33.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:32.591862+0000 mon.smithi151 (mon.0) 892 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi167.rpoanb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2023-11-16T13:36:33.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:32.594231+0000 mon.smithi151 (mon.0) 893 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi167.rpoanb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2023-11-16T13:36:33.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:32.594784+0000 mgr.smithi167.xxnzsz (mgr.14656) 53 : cephadm [INF] Ensuring nfs.foo.1 is in the ganesha grace table 2023-11-16T13:36:33.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:32.595130+0000 mon.smithi151 (mon.0) 894 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2023-11-16T13:36:33.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:32.597700+0000 mon.smithi151 (mon.0) 895 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2023-11-16T13:36:33.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:33 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:32.598797+0000 mon.smithi151 (mon.0) 896 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:34.637 INFO:teuthology.orchestra.run.smithi151.stdout: 5 86732 21.44 MB/sec warmup 113 sec latency 7065.930 ms 2023-11-16T13:36:34.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:33.242584+0000 mgr.smithi167.xxnzsz (mgr.14656) 54 : cluster [DBG] pgmap v18: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.1 MiB/s wr, 357 op/s 2023-11-16T13:36:35.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:33.242584+0000 mgr.smithi167.xxnzsz (mgr.14656) 54 : cluster [DBG] pgmap v18: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.1 MiB/s wr, 357 op/s 2023-11-16T13:36:35.637 INFO:teuthology.orchestra.run.smithi151.stdout: 5 86732 21.25 MB/sec warmup 114 sec latency 8066.024 ms 2023-11-16T13:36:35.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:35.527727+0000 mon.smithi151 (mon.0) 897 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2023-11-16T13:36:35.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:35.531723+0000 mon.smithi151 (mon.0) 898 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2023-11-16T13:36:35.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:35.539639+0000 mon.smithi167 (mon.1) 19 : audit [INF] from='mds.? [v2:172.21.15.167:6832/2149060410,v1:172.21.15.167:6833/2149060410]' entity='mds.foofs.smithi167.irznoa' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.151:0/2012699570"}]: dispatch 2023-11-16T13:36:35.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:35.540106+0000 mon.smithi151 (mon.0) 899 : audit [INF] from='mds.? ' entity='mds.foofs.smithi167.irznoa' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.151:0/2012699570"}]: dispatch 2023-11-16T13:36:35.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:35.588767+0000 mon.smithi151 (mon.0) 900 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi167.rpoanb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2023-11-16T13:36:35.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:35.592786+0000 mon.smithi151 (mon.0) 901 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi167.rpoanb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2023-11-16T13:36:35.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:35.594014+0000 mon.smithi151 (mon.0) 902 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:35.990 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:35.527727+0000 mon.smithi151 (mon.0) 897 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2023-11-16T13:36:35.990 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:35.531723+0000 mon.smithi151 (mon.0) 898 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2023-11-16T13:36:35.990 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:35.539639+0000 mon.smithi167 (mon.1) 19 : audit [INF] from='mds.? [v2:172.21.15.167:6832/2149060410,v1:172.21.15.167:6833/2149060410]' entity='mds.foofs.smithi167.irznoa' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.151:0/2012699570"}]: dispatch 2023-11-16T13:36:35.990 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:35.540106+0000 mon.smithi151 (mon.0) 899 : audit [INF] from='mds.? ' entity='mds.foofs.smithi167.irznoa' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.151:0/2012699570"}]: dispatch 2023-11-16T13:36:35.991 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:35.588767+0000 mon.smithi151 (mon.0) 900 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi167.rpoanb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2023-11-16T13:36:35.991 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:35.592786+0000 mon.smithi151 (mon.0) 901 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi167.rpoanb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2023-11-16T13:36:35.991 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:35.594014+0000 mon.smithi151 (mon.0) 902 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:36.637 INFO:teuthology.orchestra.run.smithi151.stdout: 5 86732 21.07 MB/sec warmup 115 sec latency 9066.066 ms 2023-11-16T13:36:36.891 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:35.243591+0000 mgr.smithi167.xxnzsz (mgr.14656) 55 : cluster [DBG] pgmap v19: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 2.7 MiB/s wr, 317 op/s 2023-11-16T13:36:36.891 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:35.539322+0000 mds.foofs.smithi167.irznoa (mds.0) 1 : cluster [INF] Evicting (and blocklisting) client session 14554 (172.21.15.151:0/2012699570) 2023-11-16T13:36:36.892 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:35.543259+0000 mds.foofs.smithi167.irznoa (mds.0) 2 : cluster [WRN] client session with duplicated session uuid 'ganesha-nfs.foo.0-0001' denied (client.24483 172.21.15.151:0/1249561283) 2023-11-16T13:36:36.892 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:35.588018+0000 mgr.smithi167.xxnzsz (mgr.14656) 56 : cephadm [INF] Rados config object exists: conf-nfs.foo 2023-11-16T13:36:36.892 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:35.588114+0000 mgr.smithi167.xxnzsz (mgr.14656) 57 : cephadm [INF] Creating key for client.nfs.foo.1.1.smithi167.rpoanb-rgw 2023-11-16T13:36:36.892 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:35.593360+0000 mgr.smithi167.xxnzsz (mgr.14656) 58 : cephadm [WRN] Bind address in nfs.foo.1.1.smithi167.rpoanb's ganesha conf is defaulting to empty 2023-11-16T13:36:36.892 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:35.595233+0000 mgr.smithi167.xxnzsz (mgr.14656) 59 : cephadm [INF] Deploying daemon nfs.foo.1.1.smithi167.rpoanb on smithi167 2023-11-16T13:36:36.892 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:36.533783+0000 mon.smithi151 (mon.0) 903 : audit [INF] from='mds.? ' entity='mds.foofs.smithi167.irznoa' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.151:0/2012699570"}]': finished 2023-11-16T13:36:36.892 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:36.533889+0000 mon.smithi151 (mon.0) 904 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-11-16T13:36:37.035 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:35.243591+0000 mgr.smithi167.xxnzsz (mgr.14656) 55 : cluster [DBG] pgmap v19: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 2.7 MiB/s wr, 317 op/s 2023-11-16T13:36:37.035 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:35.539322+0000 mds.foofs.smithi167.irznoa (mds.0) 1 : cluster [INF] Evicting (and blocklisting) client session 14554 (172.21.15.151:0/2012699570) 2023-11-16T13:36:37.036 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:35.543259+0000 mds.foofs.smithi167.irznoa (mds.0) 2 : cluster [WRN] client session with duplicated session uuid 'ganesha-nfs.foo.0-0001' denied (client.24483 172.21.15.151:0/1249561283) 2023-11-16T13:36:37.036 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:35.588018+0000 mgr.smithi167.xxnzsz (mgr.14656) 56 : cephadm [INF] Rados config object exists: conf-nfs.foo 2023-11-16T13:36:37.036 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:35.588114+0000 mgr.smithi167.xxnzsz (mgr.14656) 57 : cephadm [INF] Creating key for client.nfs.foo.1.1.smithi167.rpoanb-rgw 2023-11-16T13:36:37.036 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:35.593360+0000 mgr.smithi167.xxnzsz (mgr.14656) 58 : cephadm [WRN] Bind address in nfs.foo.1.1.smithi167.rpoanb's ganesha conf is defaulting to empty 2023-11-16T13:36:37.036 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:35.595233+0000 mgr.smithi167.xxnzsz (mgr.14656) 59 : cephadm [INF] Deploying daemon nfs.foo.1.1.smithi167.rpoanb on smithi167 2023-11-16T13:36:37.036 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:36.533783+0000 mon.smithi151 (mon.0) 903 : audit [INF] from='mds.? ' entity='mds.foofs.smithi167.irznoa' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.151:0/2012699570"}]': finished 2023-11-16T13:36:37.036 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:36.533889+0000 mon.smithi151 (mon.0) 904 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-11-16T13:36:37.187 INFO:teuthology.orchestra.run.smithi151.stdout:[86443] write failed on handle 11385 (Input/output error) 2023-11-16T13:36:37.187 INFO:teuthology.orchestra.run.smithi151.stdout:[86815] write failed on handle 11467 (Input/output error) 2023-11-16T13:36:37.187 INFO:teuthology.orchestra.run.smithi151.stdout:[86805] write failed on handle 11466 (Input/output error) 2023-11-16T13:36:37.187 INFO:teuthology.orchestra.run.smithi151.stdout:[86829] write failed on handle 11469 (Input/output error) 2023-11-16T13:36:37.187 INFO:teuthology.orchestra.run.smithi151.stdout:Child failed with status 1 2023-11-16T13:36:37.188 INFO:teuthology.orchestra.run.smithi151.stdout:[86770] open ./clients/client1/~dmtmp/PARADOX/__40D6B.DB failed for handle 11457 (Stale file handle) 2023-11-16T13:36:37.190 DEBUG:teuthology.orchestra.run.smithi151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2023-11-16T13:36:37.253 DEBUG:teuthology.orchestra.run.smithi151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2023-11-16T13:36:38.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:37.244099+0000 mgr.smithi167.xxnzsz (mgr.14656) 60 : cluster [DBG] pgmap v21: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 2.4 MiB/s wr, 283 op/s 2023-11-16T13:36:38.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:37.251949+0000 mon.smithi151 (mon.0) 905 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:38.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:37.256585+0000 mon.smithi151 (mon.0) 906 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:38.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:37.272673+0000 mon.smithi151 (mon.0) 907 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:38.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:37.286138+0000 mon.smithi151 (mon.0) 908 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:38.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:37.415363+0000 mon.smithi151 (mon.0) 909 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:36:38.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:37.244099+0000 mgr.smithi167.xxnzsz (mgr.14656) 60 : cluster [DBG] pgmap v21: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 2.4 MiB/s wr, 283 op/s 2023-11-16T13:36:38.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:37.251949+0000 mon.smithi151 (mon.0) 905 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:38.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:37.256585+0000 mon.smithi151 (mon.0) 906 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:38.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:37.272673+0000 mon.smithi151 (mon.0) 907 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:38.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:37.286138+0000 mon.smithi151 (mon.0) 908 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:38.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:37.415363+0000 mon.smithi151 (mon.0) 909 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:36:39.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:39 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:38.380532+0000 mon.smithi151 (mon.0) 910 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:39.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:39 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:38.380532+0000 mon.smithi151 (mon.0) 910 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:40.314 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:36:40.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:39.244516+0000 mgr.smithi167.xxnzsz (mgr.14656) 61 : cluster [DBG] pgmap v22: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.2 MiB/s wr, 260 op/s 2023-11-16T13:36:40.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:40.240633+0000 mon.smithi151 (mon.0) 911 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:40.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:40.245326+0000 mon.smithi151 (mon.0) 912 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:40.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:39.244516+0000 mgr.smithi167.xxnzsz (mgr.14656) 61 : cluster [DBG] pgmap v22: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.2 MiB/s wr, 260 op/s 2023-11-16T13:36:40.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:40.240633+0000 mon.smithi151 (mon.0) 911 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:40.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:40.245326+0000 mon.smithi151 (mon.0) 912 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:41.144198+0000 mon.smithi151 (mon.0) 913 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:41.148644+0000 mon.smithi151 (mon.0) 914 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:41.149623+0000 mon.smithi151 (mon.0) 915 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:41.150479+0000 mon.smithi151 (mon.0) 916 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:36:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:41.154863+0000 mon.smithi151 (mon.0) 917 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:41.163447+0000 mon.smithi151 (mon.0) 918 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:41.245188+0000 mgr.smithi167.xxnzsz (mgr.14656) 62 : cluster [DBG] pgmap v23: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 23 KiB/s wr, 16 op/s 2023-11-16T13:36:42.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:41.292706+0000 mgr.smithi167.xxnzsz (mgr.14656) 63 : cephadm [INF] Reconfiguring prometheus.smithi151 (dependencies changed)... 2023-11-16T13:36:42.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:36:41.303222+0000 mgr.smithi167.xxnzsz (mgr.14656) 64 : cephadm [INF] Reconfiguring daemon prometheus.smithi151 on smithi151 2023-11-16T13:36:42.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:41.144198+0000 mon.smithi151 (mon.0) 913 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:42.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:41.148644+0000 mon.smithi151 (mon.0) 914 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:41.149623+0000 mon.smithi151 (mon.0) 915 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:36:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:41.150479+0000 mon.smithi151 (mon.0) 916 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:36:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:41.154863+0000 mon.smithi151 (mon.0) 917 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:41.163447+0000 mon.smithi151 (mon.0) 918 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:36:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:41.245188+0000 mgr.smithi167.xxnzsz (mgr.14656) 62 : cluster [DBG] pgmap v23: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 23 KiB/s wr, 16 op/s 2023-11-16T13:36:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:41.292706+0000 mgr.smithi167.xxnzsz (mgr.14656) 63 : cephadm [INF] Reconfiguring prometheus.smithi151 (dependencies changed)... 2023-11-16T13:36:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:36:41.303222+0000 mgr.smithi167.xxnzsz (mgr.14656) 64 : cephadm [INF] Reconfiguring daemon prometheus.smithi151 on smithi151 2023-11-16T13:36:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:43.235305+0000 mon.smithi167 (mon.1) 20 : audit [INF] from='mds.? [v2:172.21.15.167:6832/2149060410,v1:172.21.15.167:6833/2149060410]' entity='mds.foofs.smithi167.irznoa' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.167:0/480038916"}]: dispatch 2023-11-16T13:36:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:43.235740+0000 mon.smithi151 (mon.0) 919 : audit [INF] from='mds.? ' entity='mds.foofs.smithi167.irznoa' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.167:0/480038916"}]: dispatch 2023-11-16T13:36:43.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:43.235305+0000 mon.smithi167 (mon.1) 20 : audit [INF] from='mds.? [v2:172.21.15.167:6832/2149060410,v1:172.21.15.167:6833/2149060410]' entity='mds.foofs.smithi167.irznoa' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.167:0/480038916"}]: dispatch 2023-11-16T13:36:43.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:43.235740+0000 mon.smithi151 (mon.0) 919 : audit [INF] from='mds.? ' entity='mds.foofs.smithi167.irznoa' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.167:0/480038916"}]: dispatch 2023-11-16T13:36:44.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:44 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:43.235056+0000 mds.foofs.smithi167.irznoa (mds.0) 3 : cluster [INF] Evicting (and blocklisting) client session 24327 (172.21.15.167:0/480038916) 2023-11-16T13:36:44.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:44 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:43.238912+0000 mds.foofs.smithi167.irznoa (mds.0) 4 : cluster [WRN] client session with duplicated session uuid 'ganesha-nfs.foo.1-0001' denied (client.14758 172.21.15.167:0/1529217226) 2023-11-16T13:36:44.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:44 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:43.245627+0000 mgr.smithi167.xxnzsz (mgr.14656) 65 : cluster [DBG] pgmap v24: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 23 KiB/s wr, 16 op/s 2023-11-16T13:36:44.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:44 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:43.290419+0000 mon.smithi151 (mon.0) 920 : audit [INF] from='mds.? ' entity='mds.foofs.smithi167.irznoa' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.167:0/480038916"}]': finished 2023-11-16T13:36:44.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:44 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:43.290491+0000 mon.smithi151 (mon.0) 921 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-11-16T13:36:44.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:44 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:43.381734+0000 mon.smithi151 (mon.0) 922 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:36:44.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:43.235056+0000 mds.foofs.smithi167.irznoa (mds.0) 3 : cluster [INF] Evicting (and blocklisting) client session 24327 (172.21.15.167:0/480038916) 2023-11-16T13:36:44.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:43.238912+0000 mds.foofs.smithi167.irznoa (mds.0) 4 : cluster [WRN] client session with duplicated session uuid 'ganesha-nfs.foo.1-0001' denied (client.14758 172.21.15.167:0/1529217226) 2023-11-16T13:36:44.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:43.245627+0000 mgr.smithi167.xxnzsz (mgr.14656) 65 : cluster [DBG] pgmap v24: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 23 KiB/s wr, 16 op/s 2023-11-16T13:36:44.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:43.290419+0000 mon.smithi151 (mon.0) 920 : audit [INF] from='mds.? ' entity='mds.foofs.smithi167.irznoa' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.167:0/480038916"}]': finished 2023-11-16T13:36:44.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:43.290491+0000 mon.smithi151 (mon.0) 921 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-11-16T13:36:44.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:43.381734+0000 mon.smithi151 (mon.0) 922 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:36:45.359 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:36:46.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:45.246434+0000 mgr.smithi167.xxnzsz (mgr.14656) 66 : cluster [DBG] pgmap v26: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 28 KiB/s wr, 17 op/s 2023-11-16T13:36:46.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:45.246434+0000 mgr.smithi167.xxnzsz (mgr.14656) 66 : cluster [DBG] pgmap v26: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 28 KiB/s wr, 17 op/s 2023-11-16T13:36:48.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:47.247083+0000 mgr.smithi167.xxnzsz (mgr.14656) 67 : cluster [DBG] pgmap v27: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 24 KiB/s wr, 15 op/s 2023-11-16T13:36:48.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:47.247083+0000 mgr.smithi167.xxnzsz (mgr.14656) 67 : cluster [DBG] pgmap v27: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 24 KiB/s wr, 15 op/s 2023-11-16T13:36:50.403 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:36:50.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:49.247502+0000 mgr.smithi167.xxnzsz (mgr.14656) 68 : cluster [DBG] pgmap v28: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 24 KiB/s wr, 15 op/s 2023-11-16T13:36:50.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:49.247502+0000 mgr.smithi167.xxnzsz (mgr.14656) 68 : cluster [DBG] pgmap v28: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 24 KiB/s wr, 15 op/s 2023-11-16T13:36:52.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:51.248353+0000 mgr.smithi167.xxnzsz (mgr.14656) 69 : cluster [DBG] pgmap v29: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 307 B/s rd, 2.9 KiB/s wr, 1 op/s 2023-11-16T13:36:52.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:51.248353+0000 mgr.smithi167.xxnzsz (mgr.14656) 69 : cluster [DBG] pgmap v29: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 307 B/s rd, 2.9 KiB/s wr, 1 op/s 2023-11-16T13:36:54.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:54 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:53.248825+0000 mgr.smithi167.xxnzsz (mgr.14656) 70 : cluster [DBG] pgmap v30: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 307 B/s rd, 2.9 KiB/s wr, 1 op/s 2023-11-16T13:36:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:54 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:53.248825+0000 mgr.smithi167.xxnzsz (mgr.14656) 70 : cluster [DBG] pgmap v30: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 307 B/s rd, 2.9 KiB/s wr, 1 op/s 2023-11-16T13:36:55.447 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:36:56.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:56 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:55.249743+0000 mgr.smithi167.xxnzsz (mgr.14656) 71 : cluster [DBG] pgmap v31: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 428 B/s rd, 2.5 KiB/s wr, 1 op/s 2023-11-16T13:36:56.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:56 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:55.249743+0000 mgr.smithi167.xxnzsz (mgr.14656) 71 : cluster [DBG] pgmap v31: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 428 B/s rd, 2.5 KiB/s wr, 1 op/s 2023-11-16T13:36:58.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:57.250168+0000 mgr.smithi167.xxnzsz (mgr.14656) 72 : cluster [DBG] pgmap v32: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:36:58.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:57.250168+0000 mgr.smithi167.xxnzsz (mgr.14656) 72 : cluster [DBG] pgmap v32: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:36:59.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:36:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:36:58.381812+0000 mon.smithi151 (mon.0) 923 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:36:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:36:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:36:58.381812+0000 mon.smithi151 (mon.0) 923 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:37:00.483 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:36:59.250658+0000 mgr.smithi167.xxnzsz (mgr.14656) 73 : cluster [DBG] pgmap v33: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:00.497 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:37:00.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:36:59.250658+0000 mgr.smithi167.xxnzsz (mgr.14656) 73 : cluster [DBG] pgmap v33: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:02.273 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:37:02.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:01.251431+0000 mgr.smithi167.xxnzsz (mgr.14656) 74 : cluster [DBG] pgmap v34: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:02.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:01.251431+0000 mgr.smithi167.xxnzsz (mgr.14656) 74 : cluster [DBG] pgmap v34: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:02.871 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:37:02.871 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (5m) 21s ago 6m 23.4M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:37:02.871 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (6m) 21s ago 6m 7008k - 16.2.5 6933c2a0b7dd 954bd32d4ee2 2023-11-16T13:37:02.871 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (5m) 22s ago 5m 7155k - 16.2.5 6933c2a0b7dd 649801b49aea 2023-11-16T13:37:02.871 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (5m) 21s ago 5m 35.8M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:37:02.871 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (2m) 21s ago 2m 14.4M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:37:02.871 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (2m) 22s ago 2m 41.8M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:37:02.871 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:9283 running (7m) 21s ago 7m 389M - 16.2.5 6933c2a0b7dd a009db54a97f 2023-11-16T13:37:02.872 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (59s) 22s ago 5m 493M - 17.2.7-68-gae6ce92c ea201341e8f3 d8a5483bcd6d 2023-11-16T13:37:02.872 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (7m) 21s ago 7m 121M 2048M 16.2.5 6933c2a0b7dd 3a504a936c5d 2023-11-16T13:37:02.872 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (5m) 22s ago 5m 80.2M 2048M 16.2.5 6933c2a0b7dd 448b57d6232a 2023-11-16T13:37:02.872 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (30s) 21s ago 30s 52.2M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:37:02.872 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (25s) 22s ago 25s 18.9M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:37:02.872 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (5m) 21s ago 5m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:37:02.872 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (5m) 22s ago 5m 18.2M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:37:02.872 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (5m) 21s ago 5m 1218M 1944M 16.2.5 6933c2a0b7dd 7598578fb701 2023-11-16T13:37:02.872 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (4m) 21s ago 4m 1069M 1944M 16.2.5 6933c2a0b7dd 989f26649fd3 2023-11-16T13:37:02.872 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (4m) 21s ago 4m 634M 1944M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:37:02.873 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (4m) 21s ago 4m 1145M 1944M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:37:02.873 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (4m) 22s ago 4m 1152M 2712M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:37:02.873 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (3m) 22s ago 3m 1353M 2712M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:37:02.873 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (3m) 22s ago 3m 1040M 2712M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:37:02.873 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (3m) 22s ago 3m 1801M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:37:02.873 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (4m) 21s ago 5m 48.9M - 2.18.1 de242295e225 336d5d209623 2023-11-16T13:37:03.233 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:37:03.233 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:37:03.234 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:37:03.234 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:37:03.234 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:37:03.235 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1, 2023-11-16T13:37:03.235 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 1 2023-11-16T13:37:03.235 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:37:03.236 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:37:03.236 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2023-11-16T13:37:03.236 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:37:03.236 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:37:03.237 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:37:03.237 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:37:03.237 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:37:03.237 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 13, 2023-11-16T13:37:03.238 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 1 2023-11-16T13:37:03.238 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:37:03.239 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:37:03.543 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:37:03.544 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:37:03.544 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:37:03.544 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:37:03.544 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [], 2023-11-16T13:37:03.544 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "1/23 daemons upgraded", 2023-11-16T13:37:03.544 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "", 2023-11-16T13:37:03.544 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:37:03.544 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:37:03.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:02.257679+0000 mgr.smithi167.xxnzsz (mgr.14656) 75 : audit [DBG] from='client.14762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:03.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:02.563241+0000 mgr.smithi167.xxnzsz (mgr.14656) 76 : audit [DBG] from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:03.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:02.859435+0000 mgr.smithi167.xxnzsz (mgr.14656) 77 : audit [DBG] from='client.14770 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:03.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:03.231653+0000 mon.smithi151 (mon.0) 924 : audit [DBG] from='client.? 172.21.15.151:0/1641451518' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:37:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:02.257679+0000 mgr.smithi167.xxnzsz (mgr.14656) 75 : audit [DBG] from='client.14762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:02.563241+0000 mgr.smithi167.xxnzsz (mgr.14656) 76 : audit [DBG] from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:02.859435+0000 mgr.smithi167.xxnzsz (mgr.14656) 77 : audit [DBG] from='client.14770 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:03.231653+0000 mon.smithi151 (mon.0) 924 : audit [DBG] from='client.? 172.21.15.151:0/1641451518' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:37:04.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:03.251828+0000 mgr.smithi167.xxnzsz (mgr.14656) 78 : cluster [DBG] pgmap v35: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:04.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:03.541678+0000 mgr.smithi167.xxnzsz (mgr.14656) 79 : audit [DBG] from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:04.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:03.251828+0000 mgr.smithi167.xxnzsz (mgr.14656) 78 : cluster [DBG] pgmap v35: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:04.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:03.541678+0000 mgr.smithi167.xxnzsz (mgr.14656) 79 : audit [DBG] from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:05.537 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:37:06.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:05.252471+0000 mgr.smithi167.xxnzsz (mgr.14656) 80 : cluster [DBG] pgmap v36: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:06.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:06 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:05.252471+0000 mgr.smithi167.xxnzsz (mgr.14656) 80 : cluster [DBG] pgmap v36: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:08.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:08 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:07.252842+0000 mgr.smithi167.xxnzsz (mgr.14656) 81 : cluster [DBG] pgmap v37: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:08.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:07.252842+0000 mgr.smithi167.xxnzsz (mgr.14656) 81 : cluster [DBG] pgmap v37: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:10.582 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:37:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:09.253399+0000 mgr.smithi167.xxnzsz (mgr.14656) 82 : cluster [DBG] pgmap v38: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:10.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:09.253399+0000 mgr.smithi167.xxnzsz (mgr.14656) 82 : cluster [DBG] pgmap v38: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:12.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:11.254064+0000 mgr.smithi167.xxnzsz (mgr.14656) 83 : cluster [DBG] pgmap v39: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:12.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:11.254064+0000 mgr.smithi167.xxnzsz (mgr.14656) 83 : cluster [DBG] pgmap v39: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:14.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:13.254486+0000 mgr.smithi167.xxnzsz (mgr.14656) 84 : cluster [DBG] pgmap v40: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:14.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:13.382168+0000 mon.smithi151 (mon.0) 925 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:37:14.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:13.254486+0000 mgr.smithi167.xxnzsz (mgr.14656) 84 : cluster [DBG] pgmap v40: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:14.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:13.382168+0000 mon.smithi151 (mon.0) 925 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:37:15.620 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:37:16.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:15.255381+0000 mgr.smithi167.xxnzsz (mgr.14656) 85 : cluster [DBG] pgmap v41: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:16.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:15.255381+0000 mgr.smithi167.xxnzsz (mgr.14656) 85 : cluster [DBG] pgmap v41: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:18.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:17.255737+0000 mgr.smithi167.xxnzsz (mgr.14656) 86 : cluster [DBG] pgmap v42: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:18.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:17.255737+0000 mgr.smithi167.xxnzsz (mgr.14656) 86 : cluster [DBG] pgmap v42: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:20.661 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:37:20.733 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:20 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:19.256292+0000 mgr.smithi167.xxnzsz (mgr.14656) 87 : cluster [DBG] pgmap v43: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:20.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:20 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:19.256292+0000 mgr.smithi167.xxnzsz (mgr.14656) 87 : cluster [DBG] pgmap v43: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:22.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:21.256957+0000 mgr.smithi167.xxnzsz (mgr.14656) 88 : cluster [DBG] pgmap v44: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:26.384 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:23.257513+0000 mgr.smithi167.xxnzsz (mgr.14656) 89 : cluster [DBG] pgmap v45: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:26.384 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:21.256957+0000 mgr.smithi167.xxnzsz (mgr.14656) 88 : cluster [DBG] pgmap v44: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:26.384 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:23.257513+0000 mgr.smithi167.xxnzsz (mgr.14656) 89 : cluster [DBG] pgmap v45: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:26.384 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:37:26.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:25.258248+0000 mgr.smithi167.xxnzsz (mgr.14656) 90 : cluster [DBG] pgmap v46: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:26.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:25.258248+0000 mgr.smithi167.xxnzsz (mgr.14656) 90 : cluster [DBG] pgmap v46: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:28.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:27.258720+0000 mgr.smithi167.xxnzsz (mgr.14656) 91 : cluster [DBG] pgmap v47: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:28.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:27.258720+0000 mgr.smithi167.xxnzsz (mgr.14656) 91 : cluster [DBG] pgmap v47: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:28.383664+0000 mon.smithi151 (mon.0) 926 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:37:29.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:28.383664+0000 mon.smithi151 (mon.0) 926 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:37:30.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:30 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:29.259263+0000 mgr.smithi167.xxnzsz (mgr.14656) 92 : cluster [DBG] pgmap v48: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:30.743 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:37:30.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:30 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:29.259263+0000 mgr.smithi167.xxnzsz (mgr.14656) 92 : cluster [DBG] pgmap v48: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:32.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:31.260046+0000 mgr.smithi167.xxnzsz (mgr.14656) 93 : cluster [DBG] pgmap v49: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:32.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:31.260046+0000 mgr.smithi167.xxnzsz (mgr.14656) 93 : cluster [DBG] pgmap v49: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:33.878 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:37:34.475 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:37:34.477 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (5m) 53s ago 6m 23.4M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:37:34.477 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (6m) 53s ago 6m 7008k - 16.2.5 6933c2a0b7dd 954bd32d4ee2 2023-11-16T13:37:34.477 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (6m) 54s ago 6m 7155k - 16.2.5 6933c2a0b7dd 649801b49aea 2023-11-16T13:37:34.477 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (5m) 53s ago 6m 35.8M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:37:34.477 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (3m) 53s ago 3m 14.4M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:37:34.477 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (3m) 54s ago 3m 41.8M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:37:34.477 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:9283 running (7m) 53s ago 7m 389M - 16.2.5 6933c2a0b7dd a009db54a97f 2023-11-16T13:37:34.477 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (90s) 54s ago 6m 493M - 17.2.7-68-gae6ce92c ea201341e8f3 d8a5483bcd6d 2023-11-16T13:37:34.478 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (7m) 53s ago 8m 121M 2048M 16.2.5 6933c2a0b7dd 3a504a936c5d 2023-11-16T13:37:34.478 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (6m) 54s ago 6m 80.2M 2048M 16.2.5 6933c2a0b7dd 448b57d6232a 2023-11-16T13:37:34.478 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (62s) 53s ago 61s 52.2M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:37:34.478 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (57s) 54s ago 57s 18.9M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:37:34.478 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (6m) 53s ago 6m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:37:34.478 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (5m) 54s ago 5m 18.2M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:37:34.478 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (5m) 53s ago 5m 1218M 1944M 16.2.5 6933c2a0b7dd 7598578fb701 2023-11-16T13:37:34.478 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (5m) 53s ago 5m 1069M 1944M 16.2.5 6933c2a0b7dd 989f26649fd3 2023-11-16T13:37:34.478 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (5m) 53s ago 5m 634M 1944M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:37:34.478 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (4m) 53s ago 4m 1145M 1944M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:37:34.478 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (4m) 54s ago 4m 1152M 2712M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:37:34.478 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (4m) 54s ago 4m 1353M 2712M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:37:34.479 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (4m) 54s ago 4m 1040M 2712M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:37:34.479 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (4m) 54s ago 4m 1801M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:37:34.479 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (5m) 53s ago 6m 48.9M - 2.18.1 de242295e225 336d5d209623 2023-11-16T13:37:34.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:33.260443+0000 mgr.smithi167.xxnzsz (mgr.14656) 94 : cluster [DBG] pgmap v50: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:34.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:33.864044+0000 mgr.smithi167.xxnzsz (mgr.14656) 95 : audit [DBG] from='client.14782 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:34.841 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:37:34.841 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:37:34.841 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:37:34.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:37:34.842 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:37:34.842 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1, 2023-11-16T13:37:34.842 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 1 2023-11-16T13:37:34.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:37:34.843 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:37:34.843 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2023-11-16T13:37:34.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:37:34.844 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:37:34.844 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:37:34.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:37:34.844 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:37:34.845 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 13, 2023-11-16T13:37:34.845 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 1 2023-11-16T13:37:34.845 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:37:34.845 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:37:34.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:33.260443+0000 mgr.smithi167.xxnzsz (mgr.14656) 94 : cluster [DBG] pgmap v50: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:34.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:33.864044+0000 mgr.smithi167.xxnzsz (mgr.14656) 95 : audit [DBG] from='client.14782 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:35.141 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:37:35.141 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:37:35.141 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:37:35.142 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:37:35.142 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [], 2023-11-16T13:37:35.142 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "1/23 daemons upgraded", 2023-11-16T13:37:35.142 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "", 2023-11-16T13:37:35.143 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:37:35.143 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:37:35.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:34.164209+0000 mgr.smithi167.xxnzsz (mgr.14656) 96 : audit [DBG] from='client.14786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:35.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:34.463458+0000 mgr.smithi167.xxnzsz (mgr.14656) 97 : audit [DBG] from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:35.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:35 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:34.839516+0000 mon.smithi151 (mon.0) 927 : audit [DBG] from='client.? 172.21.15.151:0/1226338427' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:37:35.778 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:37:35.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:34.164209+0000 mgr.smithi167.xxnzsz (mgr.14656) 96 : audit [DBG] from='client.14786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:35.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:34.463458+0000 mgr.smithi167.xxnzsz (mgr.14656) 97 : audit [DBG] from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:35.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:34.839516+0000 mon.smithi151 (mon.0) 927 : audit [DBG] from='client.? 172.21.15.151:0/1226338427' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:37:36.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:35.139350+0000 mgr.smithi167.xxnzsz (mgr.14656) 98 : audit [DBG] from='client.14798 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:36.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:35.261327+0000 mgr.smithi167.xxnzsz (mgr.14656) 99 : cluster [DBG] pgmap v51: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:36.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:35.139350+0000 mgr.smithi167.xxnzsz (mgr.14656) 98 : audit [DBG] from='client.14798 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:37:36.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:35.261327+0000 mgr.smithi167.xxnzsz (mgr.14656) 99 : cluster [DBG] pgmap v51: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:38.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:37.261796+0000 mgr.smithi167.xxnzsz (mgr.14656) 100 : cluster [DBG] pgmap v52: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:38.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:37.261796+0000 mgr.smithi167.xxnzsz (mgr.14656) 100 : cluster [DBG] pgmap v52: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:40.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:39.262368+0000 mgr.smithi167.xxnzsz (mgr.14656) 101 : cluster [DBG] pgmap v53: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:40.829 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:37:40.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:39.262368+0000 mgr.smithi167.xxnzsz (mgr.14656) 101 : cluster [DBG] pgmap v53: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:37:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:41.262987+0000 mgr.smithi167.xxnzsz (mgr.14656) 102 : cluster [DBG] pgmap v54: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:42.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:41.262987+0000 mgr.smithi167.xxnzsz (mgr.14656) 102 : cluster [DBG] pgmap v54: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:43.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:43.382940+0000 mon.smithi151 (mon.0) 928 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:37:43.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:43.382940+0000 mon.smithi151 (mon.0) 928 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:37:44.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:43.263476+0000 mgr.smithi167.xxnzsz (mgr.14656) 103 : cluster [DBG] pgmap v55: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:44.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:44 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:43.263476+0000 mgr.smithi167.xxnzsz (mgr.14656) 103 : cluster [DBG] pgmap v55: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:45.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:44.429918+0000 mon.smithi151 (mon.0) 929 : cluster [DBG] mgrmap e28: smithi167.xxnzsz(active, since 91s), standbys: smithi151.fcujra 2023-11-16T13:37:45.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:45 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:44.429918+0000 mon.smithi151 (mon.0) 929 : cluster [DBG] mgrmap e28: smithi167.xxnzsz(active, since 91s), standbys: smithi151.fcujra 2023-11-16T13:37:45.870 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:37:46.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:45.264271+0000 mgr.smithi167.xxnzsz (mgr.14656) 104 : cluster [DBG] pgmap v56: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 1 op/s 2023-11-16T13:37:46.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:45.264271+0000 mgr.smithi167.xxnzsz (mgr.14656) 104 : cluster [DBG] pgmap v56: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 1 op/s 2023-11-16T13:37:48.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:47.264848+0000 mgr.smithi167.xxnzsz (mgr.14656) 105 : cluster [DBG] pgmap v57: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:37:48.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:47.264848+0000 mgr.smithi167.xxnzsz (mgr.14656) 105 : cluster [DBG] pgmap v57: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:37:50.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:49.265410+0000 mgr.smithi167.xxnzsz (mgr.14656) 106 : cluster [DBG] pgmap v58: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2023-11-16T13:37:50.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:49.265410+0000 mgr.smithi167.xxnzsz (mgr.14656) 106 : cluster [DBG] pgmap v58: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2023-11-16T13:37:50.906 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:37:52.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:51.265945+0000 mgr.smithi167.xxnzsz (mgr.14656) 107 : cluster [DBG] pgmap v59: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:37:52.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:51.265945+0000 mgr.smithi167.xxnzsz (mgr.14656) 107 : cluster [DBG] pgmap v59: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:37:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:54 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:53.266493+0000 mgr.smithi167.xxnzsz (mgr.14656) 108 : cluster [DBG] pgmap v60: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:37:54.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:54 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:53.266493+0000 mgr.smithi167.xxnzsz (mgr.14656) 108 : cluster [DBG] pgmap v60: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:37:55.954 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:37:56.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:56 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:55.267330+0000 mgr.smithi167.xxnzsz (mgr.14656) 109 : cluster [DBG] pgmap v61: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 1 op/s 2023-11-16T13:37:56.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:56 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:55.267330+0000 mgr.smithi167.xxnzsz (mgr.14656) 109 : cluster [DBG] pgmap v61: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 1 op/s 2023-11-16T13:37:58.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:57.267897+0000 mgr.smithi167.xxnzsz (mgr.14656) 110 : cluster [DBG] pgmap v62: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:58.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:37:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:37:58.383242+0000 mon.smithi151 (mon.0) 930 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:37:58.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:57.267897+0000 mgr.smithi167.xxnzsz (mgr.14656) 110 : cluster [DBG] pgmap v62: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:37:58.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:37:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:37:58.383242+0000 mon.smithi151 (mon.0) 930 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:38:00.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:37:59.268472+0000 mgr.smithi167.xxnzsz (mgr.14656) 111 : cluster [DBG] pgmap v63: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:00.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:37:59.268472+0000 mgr.smithi167.xxnzsz (mgr.14656) 111 : cluster [DBG] pgmap v63: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:00.999 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:38:02.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:01.269046+0000 mgr.smithi167.xxnzsz (mgr.14656) 112 : cluster [DBG] pgmap v64: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:02.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:01.269046+0000 mgr.smithi167.xxnzsz (mgr.14656) 112 : cluster [DBG] pgmap v64: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:04.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:03.269468+0000 mgr.smithi167.xxnzsz (mgr.14656) 113 : cluster [DBG] pgmap v65: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:04.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:03.269468+0000 mgr.smithi167.xxnzsz (mgr.14656) 113 : cluster [DBG] pgmap v65: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:05.469 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:38:06.032 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:38:06.078 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:38:06.078 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (6m) 84s ago 7m 23.4M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:38:06.078 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (7m) 84s ago 7m 7008k - 16.2.5 6933c2a0b7dd 954bd32d4ee2 2023-11-16T13:38:06.078 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (6m) 85s ago 6m 7155k - 16.2.5 6933c2a0b7dd 649801b49aea 2023-11-16T13:38:06.079 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (6m) 84s ago 7m 35.8M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:38:06.079 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (3m) 84s ago 3m 14.4M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:38:06.079 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (3m) 85s ago 3m 41.8M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:38:06.079 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:9283 running (8m) 84s ago 8m 389M - 16.2.5 6933c2a0b7dd a009db54a97f 2023-11-16T13:38:06.079 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (2m) 85s ago 6m 493M - 17.2.7-68-gae6ce92c ea201341e8f3 d8a5483bcd6d 2023-11-16T13:38:06.079 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (8m) 84s ago 8m 121M 2048M 16.2.5 6933c2a0b7dd 3a504a936c5d 2023-11-16T13:38:06.079 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (6m) 85s ago 6m 80.2M 2048M 16.2.5 6933c2a0b7dd 448b57d6232a 2023-11-16T13:38:06.079 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (93s) 84s ago 93s 52.2M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:38:06.079 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (89s) 85s ago 88s 18.9M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:38:06.079 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (6m) 84s ago 6m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:38:06.079 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (6m) 85s ago 6m 18.2M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:38:06.079 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (6m) 84s ago 6m 1218M 1944M 16.2.5 6933c2a0b7dd 7598578fb701 2023-11-16T13:38:06.080 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (5m) 84s ago 5m 1069M 1944M 16.2.5 6933c2a0b7dd 989f26649fd3 2023-11-16T13:38:06.080 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (5m) 84s ago 5m 634M 1944M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:38:06.080 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (5m) 84s ago 5m 1145M 1944M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:38:06.080 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (5m) 85s ago 5m 1152M 2712M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:38:06.080 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (5m) 85s ago 5m 1353M 2712M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:38:06.080 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (4m) 85s ago 4m 1040M 2712M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:38:06.080 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (4m) 85s ago 4m 1801M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:38:06.080 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (5m) 84s ago 6m 48.9M - 2.18.1 de242295e225 336d5d209623 2023-11-16T13:38:06.442 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:38:06.442 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:38:06.442 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:38:06.443 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:38:06.443 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:38:06.443 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1, 2023-11-16T13:38:06.443 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 1 2023-11-16T13:38:06.443 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:38:06.443 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:38:06.443 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2023-11-16T13:38:06.443 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:38:06.443 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:38:06.443 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:38:06.443 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:38:06.443 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:38:06.444 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 13, 2023-11-16T13:38:06.444 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 1 2023-11-16T13:38:06.444 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:38:06.444 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:38:06.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:06 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:05.270333+0000 mgr.smithi167.xxnzsz (mgr.14656) 114 : cluster [DBG] pgmap v66: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:06.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:06 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:05.454852+0000 mgr.smithi167.xxnzsz (mgr.14656) 115 : audit [DBG] from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:06.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:06 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:05.765657+0000 mgr.smithi167.xxnzsz (mgr.14656) 116 : audit [DBG] from='client.14806 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:06.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:06 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:06.441169+0000 mon.smithi151 (mon.0) 931 : audit [DBG] from='client.? 172.21.15.151:0/589730850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:38:06.749 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:38:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:38:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:38:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:38:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [], 2023-11-16T13:38:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "1/23 daemons upgraded", 2023-11-16T13:38:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "", 2023-11-16T13:38:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:38:06.750 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:38:06.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:05.270333+0000 mgr.smithi167.xxnzsz (mgr.14656) 114 : cluster [DBG] pgmap v66: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:06.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:05.454852+0000 mgr.smithi167.xxnzsz (mgr.14656) 115 : audit [DBG] from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:06.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:05.765657+0000 mgr.smithi167.xxnzsz (mgr.14656) 116 : audit [DBG] from='client.14806 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:06.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:06.441169+0000 mon.smithi151 (mon.0) 931 : audit [DBG] from='client.? 172.21.15.151:0/589730850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:38:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:06.067251+0000 mgr.smithi167.xxnzsz (mgr.14656) 117 : audit [DBG] from='client.14810 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:06.747871+0000 mgr.smithi167.xxnzsz (mgr.14656) 118 : audit [DBG] from='client.14818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:07.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:06.067251+0000 mgr.smithi167.xxnzsz (mgr.14656) 117 : audit [DBG] from='client.14810 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:07.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:06.747871+0000 mgr.smithi167.xxnzsz (mgr.14656) 118 : audit [DBG] from='client.14818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:08.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:08 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:07.270864+0000 mgr.smithi167.xxnzsz (mgr.14656) 119 : cluster [DBG] pgmap v67: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:08.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:07.270864+0000 mgr.smithi167.xxnzsz (mgr.14656) 119 : cluster [DBG] pgmap v67: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:10.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:09.271418+0000 mgr.smithi167.xxnzsz (mgr.14656) 120 : cluster [DBG] pgmap v68: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:10.986 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:09.271418+0000 mgr.smithi167.xxnzsz (mgr.14656) 120 : cluster [DBG] pgmap v68: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:11.069 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:38:12.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:11.271920+0000 mgr.smithi167.xxnzsz (mgr.14656) 121 : cluster [DBG] pgmap v69: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:12.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:11.271920+0000 mgr.smithi167.xxnzsz (mgr.14656) 121 : cluster [DBG] pgmap v69: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:13.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:13.384776+0000 mon.smithi151 (mon.0) 932 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:38:13.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:13 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:13.384776+0000 mon.smithi151 (mon.0) 932 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:38:14.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:13.272333+0000 mgr.smithi167.xxnzsz (mgr.14656) 122 : cluster [DBG] pgmap v70: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:14.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:13.272333+0000 mgr.smithi167.xxnzsz (mgr.14656) 122 : cluster [DBG] pgmap v70: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:16.103 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:38:16.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:15.273052+0000 mgr.smithi167.xxnzsz (mgr.14656) 123 : cluster [DBG] pgmap v71: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:16.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:15.273052+0000 mgr.smithi167.xxnzsz (mgr.14656) 123 : cluster [DBG] pgmap v71: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:18.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:17.273535+0000 mgr.smithi167.xxnzsz (mgr.14656) 124 : cluster [DBG] pgmap v72: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:18.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:17.273535+0000 mgr.smithi167.xxnzsz (mgr.14656) 124 : cluster [DBG] pgmap v72: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:20.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:20 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:19.274249+0000 mgr.smithi167.xxnzsz (mgr.14656) 125 : cluster [DBG] pgmap v73: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:20.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:20 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:19.274249+0000 mgr.smithi167.xxnzsz (mgr.14656) 125 : cluster [DBG] pgmap v73: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:21.145 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:38:22.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:21.274737+0000 mgr.smithi167.xxnzsz (mgr.14656) 126 : cluster [DBG] pgmap v74: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:22.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:21.274737+0000 mgr.smithi167.xxnzsz (mgr.14656) 126 : cluster [DBG] pgmap v74: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:24.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:24 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:23.275173+0000 mgr.smithi167.xxnzsz (mgr.14656) 127 : cluster [DBG] pgmap v75: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:24.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:23.275173+0000 mgr.smithi167.xxnzsz (mgr.14656) 127 : cluster [DBG] pgmap v75: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:26.193 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:38:26.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:26 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:25.275825+0000 mgr.smithi167.xxnzsz (mgr.14656) 128 : cluster [DBG] pgmap v76: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:26.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:26 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:25.275825+0000 mgr.smithi167.xxnzsz (mgr.14656) 128 : cluster [DBG] pgmap v76: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:28.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:27.276255+0000 mgr.smithi167.xxnzsz (mgr.14656) 129 : cluster [DBG] pgmap v77: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:28.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:28.384983+0000 mon.smithi151 (mon.0) 933 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:38:28.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:27.276255+0000 mgr.smithi167.xxnzsz (mgr.14656) 129 : cluster [DBG] pgmap v77: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:28.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:28.384983+0000 mon.smithi151 (mon.0) 933 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:38:30.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:30 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:29.277053+0000 mgr.smithi167.xxnzsz (mgr.14656) 130 : cluster [DBG] pgmap v78: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:30.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:30 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:29.277053+0000 mgr.smithi167.xxnzsz (mgr.14656) 130 : cluster [DBG] pgmap v78: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:31.231 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:38:32.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:32 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:31.277462+0000 mgr.smithi167.xxnzsz (mgr.14656) 131 : cluster [DBG] pgmap v79: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:32.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:31.277462+0000 mgr.smithi167.xxnzsz (mgr.14656) 131 : cluster [DBG] pgmap v79: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:34.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:33.277963+0000 mgr.smithi167.xxnzsz (mgr.14656) 132 : cluster [DBG] pgmap v80: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:34.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:34 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:33.277963+0000 mgr.smithi167.xxnzsz (mgr.14656) 132 : cluster [DBG] pgmap v80: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:36.281 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:38:36.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:35.278873+0000 mgr.smithi167.xxnzsz (mgr.14656) 133 : cluster [DBG] pgmap v81: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:36.983 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:35.278873+0000 mgr.smithi167.xxnzsz (mgr.14656) 133 : cluster [DBG] pgmap v81: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:37.083 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:38:37.683 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:38:37.683 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (6m) 116s ago 7m 23.4M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:38:37.683 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (7m) 116s ago 7m 7008k - 16.2.5 6933c2a0b7dd 954bd32d4ee2 2023-11-16T13:38:37.683 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (7m) 117s ago 7m 7155k - 16.2.5 6933c2a0b7dd 649801b49aea 2023-11-16T13:38:37.683 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (6m) 116s ago 7m 35.8M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:38:37.684 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (4m) 116s ago 4m 14.4M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:38:37.684 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (4m) 117s ago 4m 41.8M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:38:37.684 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:9283 running (9m) 116s ago 9m 389M - 16.2.5 6933c2a0b7dd a009db54a97f 2023-11-16T13:38:37.684 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (2m) 117s ago 7m 493M - 17.2.7-68-gae6ce92c ea201341e8f3 d8a5483bcd6d 2023-11-16T13:38:37.684 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (9m) 116s ago 9m 121M 2048M 16.2.5 6933c2a0b7dd 3a504a936c5d 2023-11-16T13:38:37.684 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (7m) 117s ago 7m 80.2M 2048M 16.2.5 6933c2a0b7dd 448b57d6232a 2023-11-16T13:38:37.684 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (2m) 116s ago 2m 52.2M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:38:37.684 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (2m) 117s ago 2m 18.9M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:38:37.684 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (7m) 116s ago 7m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:38:37.684 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (6m) 117s ago 6m 18.2M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:38:37.684 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (6m) 116s ago 6m 1218M 1944M 16.2.5 6933c2a0b7dd 7598578fb701 2023-11-16T13:38:37.684 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (6m) 116s ago 6m 1069M 1944M 16.2.5 6933c2a0b7dd 989f26649fd3 2023-11-16T13:38:37.684 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (6m) 116s ago 6m 634M 1944M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:38:37.685 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (5m) 116s ago 5m 1145M 1944M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:38:37.685 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (5m) 117s ago 5m 1152M 2712M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:38:37.685 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (5m) 117s ago 5m 1353M 2712M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:38:37.685 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (5m) 117s ago 5m 1040M 2712M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:38:37.685 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (5m) 117s ago 5m 1801M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:38:37.685 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (6m) 116s ago 7m 48.9M - 2.18.1 de242295e225 336d5d209623 2023-11-16T13:38:38.044 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:38:38.044 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:38:38.044 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:38:38.044 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:38:38.044 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:38:38.044 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1, 2023-11-16T13:38:38.044 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 1 2023-11-16T13:38:38.044 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:38:38.044 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:38:38.044 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2023-11-16T13:38:38.045 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:38:38.045 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:38:38.045 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:38:38.045 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:38:38.045 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:38:38.045 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 13, 2023-11-16T13:38:38.045 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 1 2023-11-16T13:38:38.045 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:38:38.045 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:38:38.347 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:38:38.348 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:38:38.348 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:38:38.348 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:38:38.348 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [], 2023-11-16T13:38:38.348 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "1/23 daemons upgraded", 2023-11-16T13:38:38.348 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "", 2023-11-16T13:38:38.348 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:38:38.348 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:38:38.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:37.068927+0000 mgr.smithi167.xxnzsz (mgr.14656) 134 : audit [DBG] from='client.14822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:38.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:37.279314+0000 mgr.smithi167.xxnzsz (mgr.14656) 135 : cluster [DBG] pgmap v82: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:38.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:37.375314+0000 mgr.smithi167.xxnzsz (mgr.14656) 136 : audit [DBG] from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:38.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:37.670701+0000 mgr.smithi167.xxnzsz (mgr.14656) 137 : audit [DBG] from='client.14830 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:38.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:38 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:38.042565+0000 mon.smithi151 (mon.0) 934 : audit [DBG] from='client.? 172.21.15.151:0/787256454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:38:38.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:37.068927+0000 mgr.smithi167.xxnzsz (mgr.14656) 134 : audit [DBG] from='client.14822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:38.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:37.279314+0000 mgr.smithi167.xxnzsz (mgr.14656) 135 : cluster [DBG] pgmap v82: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:38.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:37.375314+0000 mgr.smithi167.xxnzsz (mgr.14656) 136 : audit [DBG] from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:38.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:37.670701+0000 mgr.smithi167.xxnzsz (mgr.14656) 137 : audit [DBG] from='client.14830 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:38.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:38.042565+0000 mon.smithi151 (mon.0) 934 : audit [DBG] from='client.? 172.21.15.151:0/787256454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:38:39.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:39 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:38.345899+0000 mgr.smithi167.xxnzsz (mgr.14656) 138 : audit [DBG] from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:39.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:39 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:38.345899+0000 mgr.smithi167.xxnzsz (mgr.14656) 138 : audit [DBG] from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:38:40.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:39.280193+0000 mgr.smithi167.xxnzsz (mgr.14656) 139 : cluster [DBG] pgmap v83: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:40.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:40 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:39.280193+0000 mgr.smithi167.xxnzsz (mgr.14656) 139 : cluster [DBG] pgmap v83: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:41.318 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:38:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:42 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:41.280745+0000 mgr.smithi167.xxnzsz (mgr.14656) 140 : cluster [DBG] pgmap v84: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:42.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:42 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:41.280745+0000 mgr.smithi167.xxnzsz (mgr.14656) 140 : cluster [DBG] pgmap v84: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:43.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:43 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:43.385381+0000 mon.smithi151 (mon.0) 935 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:38:43.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:43.385381+0000 mon.smithi151 (mon.0) 935 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:38:44.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:44 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:43.281286+0000 mgr.smithi167.xxnzsz (mgr.14656) 141 : cluster [DBG] pgmap v85: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:44.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:44 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:43.281286+0000 mgr.smithi167.xxnzsz (mgr.14656) 141 : cluster [DBG] pgmap v85: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:46.359 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:38:46.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:45.282065+0000 mgr.smithi167.xxnzsz (mgr.14656) 142 : cluster [DBG] pgmap v86: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:46.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:45.282065+0000 mgr.smithi167.xxnzsz (mgr.14656) 142 : cluster [DBG] pgmap v86: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:48.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:47.282478+0000 mgr.smithi167.xxnzsz (mgr.14656) 143 : cluster [DBG] pgmap v87: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:48.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:48.224291+0000 mon.smithi151 (mon.0) 936 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:48.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:48.229649+0000 mon.smithi151 (mon.0) 937 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:48.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:48.234389+0000 mon.smithi151 (mon.0) 938 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-11-16T13:38:48.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:48.235948+0000 mon.smithi151 (mon.0) 939 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-11-16T13:38:48.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:48.240548+0000 mon.smithi151 (mon.0) 940 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:48.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:48.245345+0000 mon.smithi151 (mon.0) 941 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:38:48.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:48 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:48.251648+0000 mon.smithi151 (mon.0) 942 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:48.919 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:47.282478+0000 mgr.smithi167.xxnzsz (mgr.14656) 143 : cluster [DBG] pgmap v87: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:48.919 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:48.224291+0000 mon.smithi151 (mon.0) 936 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:48.919 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:48.229649+0000 mon.smithi151 (mon.0) 937 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:48.919 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:48.234389+0000 mon.smithi151 (mon.0) 938 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-11-16T13:38:48.920 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:48.235948+0000 mon.smithi151 (mon.0) 939 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-11-16T13:38:48.920 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:48.240548+0000 mon.smithi151 (mon.0) 940 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:48.920 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:48.245345+0000 mon.smithi151 (mon.0) 941 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:38:48.920 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:48 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:48.251648+0000 mon.smithi151 (mon.0) 942 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:49.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:48.234817+0000 mgr.smithi167.xxnzsz (mgr.14656) 144 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-11-16T13:38:49.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:48.236254+0000 mgr.smithi167.xxnzsz (mgr.14656) 145 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-11-16T13:38:49.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:38:48.252991+0000 mgr.smithi167.xxnzsz (mgr.14656) 146 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:38:49.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:38:48.253512+0000 mgr.smithi167.xxnzsz (mgr.14656) 147 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:38:49.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:49.275087+0000 mon.smithi151 (mon.0) 943 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:49.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:49.275969+0000 mon.smithi151 (mon.0) 944 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi151.fcujra", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:38:49.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:49.276767+0000 mon.smithi151 (mon.0) 945 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:38:49.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:49 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:49.277670+0000 mon.smithi151 (mon.0) 946 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:38:49.939 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:48.234817+0000 mgr.smithi167.xxnzsz (mgr.14656) 144 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-11-16T13:38:49.940 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:48.236254+0000 mgr.smithi167.xxnzsz (mgr.14656) 145 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-11-16T13:38:49.940 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:38:48.252991+0000 mgr.smithi167.xxnzsz (mgr.14656) 146 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:38:49.940 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:38:48.253512+0000 mgr.smithi167.xxnzsz (mgr.14656) 147 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:38:49.940 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:49.275087+0000 mon.smithi151 (mon.0) 943 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:49.940 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:49.275969+0000 mon.smithi151 (mon.0) 944 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi151.fcujra", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:38:49.940 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:49.276767+0000 mon.smithi151 (mon.0) 945 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:38:49.940 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:49.277670+0000 mon.smithi151 (mon.0) 946 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:38:50.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:38:49.270277+0000 mgr.smithi167.xxnzsz (mgr.14656) 148 : cephadm [INF] Upgrade: Updating mgr.smithi151.fcujra 2023-11-16T13:38:50.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:38:49.278557+0000 mgr.smithi167.xxnzsz (mgr.14656) 149 : cephadm [INF] Deploying daemon mgr.smithi151.fcujra on smithi151 2023-11-16T13:38:50.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:50 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:49.283139+0000 mgr.smithi167.xxnzsz (mgr.14656) 150 : cluster [DBG] pgmap v88: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:50.892 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:38:49.270277+0000 mgr.smithi167.xxnzsz (mgr.14656) 148 : cephadm [INF] Upgrade: Updating mgr.smithi151.fcujra 2023-11-16T13:38:50.893 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:38:49.278557+0000 mgr.smithi167.xxnzsz (mgr.14656) 149 : cephadm [INF] Deploying daemon mgr.smithi151.fcujra on smithi151 2023-11-16T13:38:50.893 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:49.283139+0000 mgr.smithi167.xxnzsz (mgr.14656) 150 : cluster [DBG] pgmap v88: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:51.404 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:38:52.627 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:51.283620+0000 mgr.smithi167.xxnzsz (mgr.14656) 151 : cluster [DBG] pgmap v89: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:52.627 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:51.446003+0000 mon.smithi151 (mon.0) 947 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:52.627 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:51.451097+0000 mon.smithi151 (mon.0) 948 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:52.627 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:52 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:51.498167+0000 mon.smithi151 (mon.0) 949 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:38:52.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:51.283620+0000 mgr.smithi167.xxnzsz (mgr.14656) 151 : cluster [DBG] pgmap v89: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:52.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:51.446003+0000 mon.smithi151 (mon.0) 947 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:52.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:51.451097+0000 mon.smithi151 (mon.0) 948 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:52.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:52 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:51.498167+0000 mon.smithi151 (mon.0) 949 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:38:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:54 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:53.284114+0000 mgr.smithi167.xxnzsz (mgr.14656) 152 : cluster [DBG] pgmap v90: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:54 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:54.273417+0000 mon.smithi151 (mon.0) 950 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:54 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:54.278023+0000 mon.smithi151 (mon.0) 951 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:54.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:54 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:53.284114+0000 mgr.smithi167.xxnzsz (mgr.14656) 152 : cluster [DBG] pgmap v90: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:54.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:54 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:54.273417+0000 mon.smithi151 (mon.0) 950 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:54.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:54 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:54.278023+0000 mon.smithi151 (mon.0) 951 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:56.447 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:38:57.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:56 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:55.284794+0000 mgr.smithi167.xxnzsz (mgr.14656) 153 : cluster [DBG] pgmap v91: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:57.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:56 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:55.829657+0000 mon.smithi151 (mon.0) 952 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:57.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:56 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:55.834946+0000 mon.smithi151 (mon.0) 953 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:57.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:56 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:56.570101+0000 mon.smithi151 (mon.0) 954 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:57.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:56 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:56.575641+0000 mon.smithi151 (mon.0) 955 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:57.184 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:56 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:55.284794+0000 mgr.smithi167.xxnzsz (mgr.14656) 153 : cluster [DBG] pgmap v91: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:38:57.185 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:56 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:55.829657+0000 mon.smithi151 (mon.0) 952 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:57.185 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:56 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:55.834946+0000 mon.smithi151 (mon.0) 953 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:57.185 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:56 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:56.570101+0000 mon.smithi151 (mon.0) 954 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:57.185 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:56 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:56.575641+0000 mon.smithi151 (mon.0) 955 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:38:58.957 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:57.285285+0000 mgr.smithi167.xxnzsz (mgr.14656) 154 : cluster [DBG] pgmap v92: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:58.958 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:58.385699+0000 mon.smithi151 (mon.0) 956 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:38:58.958 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:58.529740+0000 mon.smithi151 (mon.0) 957 : cluster [DBG] Standby manager daemon smithi151.fcujra restarted 2023-11-16T13:38:58.958 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:58.529848+0000 mon.smithi151 (mon.0) 958 : cluster [DBG] Standby manager daemon smithi151.fcujra started 2023-11-16T13:38:58.958 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:58.532215+0000 mon.smithi151 (mon.0) 959 : audit [DBG] from='mgr.? 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi151.fcujra/crt"}]: dispatch 2023-11-16T13:38:58.958 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:58.532664+0000 mon.smithi151 (mon.0) 960 : audit [DBG] from='mgr.? 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-16T13:38:58.958 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:58.534769+0000 mon.smithi151 (mon.0) 961 : audit [DBG] from='mgr.? 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi151.fcujra/key"}]: dispatch 2023-11-16T13:38:58.958 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:58 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:58.535642+0000 mon.smithi151 (mon.0) 962 : audit [DBG] from='mgr.? 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:38:59.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:57.285285+0000 mgr.smithi167.xxnzsz (mgr.14656) 154 : cluster [DBG] pgmap v92: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:38:59.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:58.385699+0000 mon.smithi151 (mon.0) 956 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:38:59.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:58.529740+0000 mon.smithi151 (mon.0) 957 : cluster [DBG] Standby manager daemon smithi151.fcujra restarted 2023-11-16T13:38:59.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:58.529848+0000 mon.smithi151 (mon.0) 958 : cluster [DBG] Standby manager daemon smithi151.fcujra started 2023-11-16T13:38:59.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:58.532215+0000 mon.smithi151 (mon.0) 959 : audit [DBG] from='mgr.? 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi151.fcujra/crt"}]: dispatch 2023-11-16T13:38:59.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:58.532664+0000 mon.smithi151 (mon.0) 960 : audit [DBG] from='mgr.? 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-16T13:38:59.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:58.534769+0000 mon.smithi151 (mon.0) 961 : audit [DBG] from='mgr.? 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi151.fcujra/key"}]: dispatch 2023-11-16T13:38:59.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:58.535642+0000 mon.smithi151 (mon.0) 962 : audit [DBG] from='mgr.? 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:39:00.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:58.839396+0000 mon.smithi151 (mon.0) 963 : cluster [DBG] mgrmap e29: smithi167.xxnzsz(active, since 2m), standbys: smithi151.fcujra 2023-11-16T13:39:00.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:59.055130+0000 mon.smithi151 (mon.0) 964 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:39:00.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:59.061392+0000 mon.smithi151 (mon.0) 965 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:39:00.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:59.062518+0000 mon.smithi151 (mon.0) 966 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:00.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:59.063653+0000 mon.smithi151 (mon.0) 967 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:39:00.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:59.070487+0000 mon.smithi151 (mon.0) 968 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:39:00.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:59.079395+0000 mon.smithi151 (mon.0) 969 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:39:00.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:59.210589+0000 mon.smithi151 (mon.0) 970 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:00.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:38:59.215375+0000 mon.smithi151 (mon.0) 971 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mgr fail", "who": "smithi167.xxnzsz"}]: dispatch 2023-11-16T13:39:00.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:38:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:59.223460+0000 mon.smithi151 (mon.0) 972 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-11-16T13:39:00.097 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:58.839396+0000 mon.smithi151 (mon.0) 963 : cluster [DBG] mgrmap e29: smithi167.xxnzsz(active, since 2m), standbys: smithi151.fcujra 2023-11-16T13:39:00.097 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:59.055130+0000 mon.smithi151 (mon.0) 964 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:39:00.097 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:59.061392+0000 mon.smithi151 (mon.0) 965 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:39:00.098 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:59.062518+0000 mon.smithi151 (mon.0) 966 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:00.098 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:59.063653+0000 mon.smithi151 (mon.0) 967 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:39:00.098 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:59.070487+0000 mon.smithi151 (mon.0) 968 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:39:00.098 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:59.079395+0000 mon.smithi151 (mon.0) 969 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' 2023-11-16T13:39:00.098 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:59.210589+0000 mon.smithi151 (mon.0) 970 : audit [DBG] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:00.098 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:38:59.215375+0000 mon.smithi151 (mon.0) 971 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "mgr fail", "who": "smithi167.xxnzsz"}]: dispatch 2023-11-16T13:39:00.098 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:38:59 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:59.223460+0000 mon.smithi151 (mon.0) 972 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-11-16T13:39:01.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:38:59.212446+0000 mgr.smithi167.xxnzsz (mgr.14656) 155 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:39:01.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:38:59.212897+0000 mgr.smithi167.xxnzsz (mgr.14656) 156 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:39:01.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:38:59.215125+0000 mgr.smithi167.xxnzsz (mgr.14656) 157 : cephadm [INF] Failing over to other MGR 2023-11-16T13:39:01.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:38:59.285947+0000 mgr.smithi167.xxnzsz (mgr.14656) 158 : cluster [DBG] pgmap v94: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:39:01.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:00.027284+0000 mon.smithi151 (mon.0) 973 : cluster [DBG] Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:01.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.089097+0000 mon.smithi151 (mon.0) 974 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "mgr fail", "who": "smithi167.xxnzsz"}]': finished 2023-11-16T13:39:01.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:00.089168+0000 mon.smithi151 (mon.0) 975 : cluster [DBG] mgrmap e30: smithi151.fcujra(active, starting, since 0.8731s), standbys: smithi167.xxnzsz 2023-11-16T13:39:01.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.090529+0000 mon.smithi151 (mon.0) 976 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:39:01.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.090678+0000 mon.smithi151 (mon.0) 977 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:39:01.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.091480+0000 mon.smithi151 (mon.0) 978 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi151.mdkcfw"}]: dispatch 2023-11-16T13:39:01.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.091696+0000 mon.smithi151 (mon.0) 979 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:39:01.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.092103+0000 mon.smithi151 (mon.0) 980 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi151.fcujra", "id": "smithi151.fcujra"}]: dispatch 2023-11-16T13:39:01.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.092344+0000 mon.smithi151 (mon.0) 981 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi167.xxnzsz", "id": "smithi167.xxnzsz"}]: dispatch 2023-11-16T13:39:01.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.092582+0000 mon.smithi151 (mon.0) 982 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:39:01.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.092817+0000 mon.smithi151 (mon.0) 983 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:39:01.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.093025+0000 mon.smithi151 (mon.0) 984 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:39:01.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.093211+0000 mon.smithi151 (mon.0) 985 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:39:01.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.093435+0000 mon.smithi151 (mon.0) 986 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:39:01.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.093638+0000 mon.smithi151 (mon.0) 987 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:39:01.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.093956+0000 mon.smithi151 (mon.0) 988 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:39:01.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.094144+0000 mon.smithi151 (mon.0) 989 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:39:01.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.094341+0000 mon.smithi151 (mon.0) 990 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-16T13:39:01.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.094544+0000 mon.smithi151 (mon.0) 991 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-16T13:39:01.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.094961+0000 mon.smithi151 (mon.0) 992 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-16T13:39:01.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:00.127150+0000 mon.smithi151 (mon.0) 993 : cluster [INF] Manager daemon smithi151.fcujra is now available 2023-11-16T13:39:01.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.149996+0000 mon.smithi151 (mon.0) 994 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:01.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.152225+0000 mon.smithi151 (mon.0) 995 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:01.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.176018+0000 mon.smithi151 (mon.0) 996 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:39:01.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.181957+0000 mon.smithi151 (mon.0) 997 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:39:01.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.189181+0000 mon.smithi151 (mon.0) 998 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:39:01.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:00.701253+0000 mon.smithi151 (mon.0) 999 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:01.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:38:59.212446+0000 mgr.smithi167.xxnzsz (mgr.14656) 155 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:39:01.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:38:59.212897+0000 mgr.smithi167.xxnzsz (mgr.14656) 156 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:39:01.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:38:59.215125+0000 mgr.smithi167.xxnzsz (mgr.14656) 157 : cephadm [INF] Failing over to other MGR 2023-11-16T13:39:01.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:38:59.285947+0000 mgr.smithi167.xxnzsz (mgr.14656) 158 : cluster [DBG] pgmap v94: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:39:01.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:00.027284+0000 mon.smithi151 (mon.0) 973 : cluster [DBG] Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:01.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.089097+0000 mon.smithi151 (mon.0) 974 : audit [INF] from='mgr.14656 172.21.15.167:0/252206662' entity='mgr.smithi167.xxnzsz' cmd='[{"prefix": "mgr fail", "who": "smithi167.xxnzsz"}]': finished 2023-11-16T13:39:01.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:00.089168+0000 mon.smithi151 (mon.0) 975 : cluster [DBG] mgrmap e30: smithi151.fcujra(active, starting, since 0.8731s), standbys: smithi167.xxnzsz 2023-11-16T13:39:01.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.090529+0000 mon.smithi151 (mon.0) 976 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:39:01.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.090678+0000 mon.smithi151 (mon.0) 977 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:39:01.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.091480+0000 mon.smithi151 (mon.0) 978 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi151.mdkcfw"}]: dispatch 2023-11-16T13:39:01.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.091696+0000 mon.smithi151 (mon.0) 979 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:39:01.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.092103+0000 mon.smithi151 (mon.0) 980 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi151.fcujra", "id": "smithi151.fcujra"}]: dispatch 2023-11-16T13:39:01.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.092344+0000 mon.smithi151 (mon.0) 981 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi167.xxnzsz", "id": "smithi167.xxnzsz"}]: dispatch 2023-11-16T13:39:01.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.092582+0000 mon.smithi151 (mon.0) 982 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:39:01.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.092817+0000 mon.smithi151 (mon.0) 983 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:39:01.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.093025+0000 mon.smithi151 (mon.0) 984 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:39:01.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.093211+0000 mon.smithi151 (mon.0) 985 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:39:01.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.093435+0000 mon.smithi151 (mon.0) 986 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:39:01.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.093638+0000 mon.smithi151 (mon.0) 987 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:39:01.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.093956+0000 mon.smithi151 (mon.0) 988 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:39:01.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.094144+0000 mon.smithi151 (mon.0) 989 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:39:01.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.094341+0000 mon.smithi151 (mon.0) 990 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-16T13:39:01.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.094544+0000 mon.smithi151 (mon.0) 991 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-16T13:39:01.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.094961+0000 mon.smithi151 (mon.0) 992 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-16T13:39:01.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:00.127150+0000 mon.smithi151 (mon.0) 993 : cluster [INF] Manager daemon smithi151.fcujra is now available 2023-11-16T13:39:01.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.149996+0000 mon.smithi151 (mon.0) 994 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:01.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.152225+0000 mon.smithi151 (mon.0) 995 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:01.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.176018+0000 mon.smithi151 (mon.0) 996 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:39:01.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.181957+0000 mon.smithi151 (mon.0) 997 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:39:01.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.189181+0000 mon.smithi151 (mon.0) 998 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:39:01.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:00 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:00.701253+0000 mon.smithi151 (mon.0) 999 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:01.486 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:39:02.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:00.582775+0000 mgr.smithi151.fcujra (mgr.14842) 1 : cephadm [INF] [16/Nov/2023:13:39:00] ENGINE Bus STARTING 2023-11-16T13:39:02.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:00.696907+0000 mgr.smithi151.fcujra (mgr.14842) 2 : cephadm [INF] [16/Nov/2023:13:39:00] ENGINE Serving on https://172.21.15.151:7150 2023-11-16T13:39:02.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:00.697023+0000 mgr.smithi151.fcujra (mgr.14842) 3 : cephadm [INF] [16/Nov/2023:13:39:00] ENGINE Bus STARTED 2023-11-16T13:39:02.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:01.090792+0000 mon.smithi151 (mon.0) 1000 : cluster [DBG] mgrmap e31: smithi151.fcujra(active, since 1.87472s), standbys: smithi167.xxnzsz 2023-11-16T13:39:02.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:01.107568+0000 mgr.smithi151.fcujra (mgr.14842) 4 : cluster [DBG] pgmap v3: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:02.347 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:00.582775+0000 mgr.smithi151.fcujra (mgr.14842) 1 : cephadm [INF] [16/Nov/2023:13:39:00] ENGINE Bus STARTING 2023-11-16T13:39:02.347 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:00.696907+0000 mgr.smithi151.fcujra (mgr.14842) 2 : cephadm [INF] [16/Nov/2023:13:39:00] ENGINE Serving on https://172.21.15.151:7150 2023-11-16T13:39:02.347 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:00.697023+0000 mgr.smithi151.fcujra (mgr.14842) 3 : cephadm [INF] [16/Nov/2023:13:39:00] ENGINE Bus STARTED 2023-11-16T13:39:02.347 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:01.090792+0000 mon.smithi151 (mon.0) 1000 : cluster [DBG] mgrmap e31: smithi151.fcujra(active, since 1.87472s), standbys: smithi167.xxnzsz 2023-11-16T13:39:02.347 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:02 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:01.107568+0000 mgr.smithi151.fcujra (mgr.14842) 4 : cluster [DBG] pgmap v3: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:03.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:02.093818+0000 mgr.smithi151.fcujra (mgr.14842) 5 : cluster [DBG] pgmap v4: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:03 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:02.093818+0000 mgr.smithi151.fcujra (mgr.14842) 5 : cluster [DBG] pgmap v4: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:04.455 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:03.357851+0000 mon.smithi151 (mon.0) 1001 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:04.455 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:03.361574+0000 mon.smithi151 (mon.0) 1002 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:04.455 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:04.150389+0000 mon.smithi151 (mon.0) 1003 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:04.455 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:04.154258+0000 mon.smithi151 (mon.0) 1004 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:04.455 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:04.323233+0000 mon.smithi151 (mon.0) 1005 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:04.455 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:04.328315+0000 mon.smithi151 (mon.0) 1006 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:04.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:03.357851+0000 mon.smithi151 (mon.0) 1001 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:04.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:03.361574+0000 mon.smithi151 (mon.0) 1002 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:04.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:04.150389+0000 mon.smithi151 (mon.0) 1003 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:04.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:04.154258+0000 mon.smithi151 (mon.0) 1004 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:04.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:04.323233+0000 mon.smithi151 (mon.0) 1005 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:04.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:04 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:04.328315+0000 mon.smithi151 (mon.0) 1006 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:05.657 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:04.094224+0000 mgr.smithi151.fcujra (mgr.14842) 6 : cluster [DBG] pgmap v5: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:05.657 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:05.066611+0000 mon.smithi151 (mon.0) 1007 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:05.657 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:05 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:05.070631+0000 mon.smithi151 (mon.0) 1008 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:05.661 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:04.094224+0000 mgr.smithi151.fcujra (mgr.14842) 6 : cluster [DBG] pgmap v5: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:05.661 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:05.066611+0000 mon.smithi151 (mon.0) 1007 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:05.661 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:05.070631+0000 mon.smithi151 (mon.0) 1008 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:06.536 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:39:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:06.094684+0000 mgr.smithi151.fcujra (mgr.14842) 7 : cluster [DBG] pgmap v6: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:06.938591+0000 mon.smithi151 (mon.0) 1009 : cluster [DBG] Standby manager daemon smithi167.xxnzsz restarted 2023-11-16T13:39:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:06.938731+0000 mon.smithi151 (mon.0) 1010 : cluster [DBG] Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:06.940206+0000 mon.smithi167 (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.167:0/4114684901' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/crt"}]: dispatch 2023-11-16T13:39:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:06.940544+0000 mon.smithi167 (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.167:0/4114684901' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-16T13:39:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:06.942899+0000 mon.smithi167 (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.167:0/4114684901' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/key"}]: dispatch 2023-11-16T13:39:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:06.943187+0000 mon.smithi167 (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.167:0/4114684901' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:39:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:07.345401+0000 mon.smithi151 (mon.0) 1011 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:07.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:07.351948+0000 mon.smithi151 (mon.0) 1012 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:07.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:07.353067+0000 mon.smithi151 (mon.0) 1013 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:07.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:07 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:07.353710+0000 mon.smithi151 (mon.0) 1014 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:39:07.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:06.094684+0000 mgr.smithi151.fcujra (mgr.14842) 7 : cluster [DBG] pgmap v6: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:07.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:06.938591+0000 mon.smithi151 (mon.0) 1009 : cluster [DBG] Standby manager daemon smithi167.xxnzsz restarted 2023-11-16T13:39:07.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:06.938731+0000 mon.smithi151 (mon.0) 1010 : cluster [DBG] Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:07.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:06.940206+0000 mon.smithi167 (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.167:0/4114684901' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/crt"}]: dispatch 2023-11-16T13:39:07.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:06.940544+0000 mon.smithi167 (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.167:0/4114684901' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-16T13:39:07.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:06.942899+0000 mon.smithi167 (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.167:0/4114684901' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/key"}]: dispatch 2023-11-16T13:39:07.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:06.943187+0000 mon.smithi167 (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.167:0/4114684901' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:39:07.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:07.345401+0000 mon.smithi151 (mon.0) 1011 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:07.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:07.351948+0000 mon.smithi151 (mon.0) 1012 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:07.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:07.353067+0000 mon.smithi151 (mon.0) 1013 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:07.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:07.353710+0000 mon.smithi151 (mon.0) 1014 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:39:08.679 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:39:08.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:08 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:07.354548+0000 mgr.smithi151.fcujra (mgr.14842) 8 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-11-16T13:39:08.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:08 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:07.354701+0000 mgr.smithi151.fcujra (mgr.14842) 9 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.conf 2023-11-16T13:39:08.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:08 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:07.372296+0000 mon.smithi151 (mon.0) 1015 : cluster [DBG] mgrmap e32: smithi151.fcujra(active, since 8s), standbys: smithi167.xxnzsz 2023-11-16T13:39:08.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:07.354548+0000 mgr.smithi151.fcujra (mgr.14842) 8 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-11-16T13:39:08.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:07.354701+0000 mgr.smithi151.fcujra (mgr.14842) 9 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.conf 2023-11-16T13:39:08.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:07.372296+0000 mon.smithi151 (mon.0) 1015 : cluster [DBG] mgrmap e32: smithi151.fcujra(active, since 8s), standbys: smithi167.xxnzsz 2023-11-16T13:39:09.285 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:39:09.286 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (7m) 4s ago 8m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:39:09.286 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (8m) 4s ago 8m 7008k - 16.2.5 6933c2a0b7dd 954bd32d4ee2 2023-11-16T13:39:09.286 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (7m) 5s ago 7m 7159k - 16.2.5 6933c2a0b7dd 649801b49aea 2023-11-16T13:39:09.286 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (7m) 4s ago 8m 36.0M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:39:09.286 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (4m) 4s ago 4m 15.2M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:39:09.286 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (4m) 5s ago 4m 42.0M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:39:09.287 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (18s) 4s ago 9m 471M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:39:09.287 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (3m) 5s ago 7m 73.0M - 17.2.7-68-gae6ce92c ea201341e8f3 d8a5483bcd6d 2023-11-16T13:39:09.287 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (9m) 4s ago 9m 136M 2048M 16.2.5 6933c2a0b7dd 3a504a936c5d 2023-11-16T13:39:09.287 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (7m) 5s ago 7m 85.7M 2048M 16.2.5 6933c2a0b7dd 448b57d6232a 2023-11-16T13:39:09.287 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (2m) 4s ago 2m 65.7M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:39:09.288 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (2m) 5s ago 2m 56.0M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:39:09.288 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (7m) 4s ago 7m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:39:09.288 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (7m) 5s ago 7m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:39:09.288 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (7m) 4s ago 7m 1226M 1944M 16.2.5 6933c2a0b7dd 7598578fb701 2023-11-16T13:39:09.288 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (6m) 4s ago 6m 1077M 1944M 16.2.5 6933c2a0b7dd 989f26649fd3 2023-11-16T13:39:09.288 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (6m) 4s ago 6m 641M 1944M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:39:09.289 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (6m) 4s ago 6m 1157M 1944M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:39:09.289 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (6m) 5s ago 6m 1160M 2712M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:39:09.289 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (6m) 5s ago 6m 1360M 2712M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:39:09.289 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (5m) 5s ago 5m 1047M 2712M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:39:09.289 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (5m) 5s ago 5m 1811M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:39:09.289 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (21s) 4s ago 7m 42.5M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:39:09.455 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:07.619195+0000 mgr.smithi151.fcujra (mgr.14842) 10 : cephadm [INF] Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:09.455 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:07.619339+0000 mgr.smithi151.fcujra (mgr.14842) 11 : cephadm [INF] Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:09.456 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:07.883435+0000 mgr.smithi151.fcujra (mgr.14842) 12 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:09.456 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:07.884068+0000 mgr.smithi151.fcujra (mgr.14842) 13 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:09.456 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:08.095213+0000 mgr.smithi151.fcujra (mgr.14842) 14 : cluster [DBG] pgmap v7: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 146 B/s rd, 146 B/s wr, 0 op/s 2023-11-16T13:39:09.456 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:08.147853+0000 mgr.smithi151.fcujra (mgr.14842) 15 : cephadm [INF] Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:09.456 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:08.148497+0000 mgr.smithi151.fcujra (mgr.14842) 16 : cephadm [INF] Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:09.456 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:08.416100+0000 mon.smithi151 (mon.0) 1016 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:09.456 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:08.419206+0000 mon.smithi151 (mon.0) 1017 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:09.456 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:08.422208+0000 mon.smithi151 (mon.0) 1018 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:09.456 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:08.426086+0000 mon.smithi151 (mon.0) 1019 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:09.456 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:08.432193+0000 mon.smithi151 (mon.0) 1020 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:09.456 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:08.440850+0000 mon.smithi151 (mon.0) 1021 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:09.457 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:08.539518+0000 mon.smithi151 (mon.0) 1022 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:09.653 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:39:09.653 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:39:09.653 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:39:09.653 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:39:09.653 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:39:09.653 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:39:09.653 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:39:09.654 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:39:09.654 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2023-11-16T13:39:09.654 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:39:09.654 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:39:09.654 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:39:09.654 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:39:09.654 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:39:09.654 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 12, 2023-11-16T13:39:09.654 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:39:09.654 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:39:09.654 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:39:09.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:07.619195+0000 mgr.smithi151.fcujra (mgr.14842) 10 : cephadm [INF] Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:09.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:07.619339+0000 mgr.smithi151.fcujra (mgr.14842) 11 : cephadm [INF] Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:09.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:07.883435+0000 mgr.smithi151.fcujra (mgr.14842) 12 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:09.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:07.884068+0000 mgr.smithi151.fcujra (mgr.14842) 13 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:09.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:08.095213+0000 mgr.smithi151.fcujra (mgr.14842) 14 : cluster [DBG] pgmap v7: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 146 B/s rd, 146 B/s wr, 0 op/s 2023-11-16T13:39:09.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:08.147853+0000 mgr.smithi151.fcujra (mgr.14842) 15 : cephadm [INF] Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:09.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:08.148497+0000 mgr.smithi151.fcujra (mgr.14842) 16 : cephadm [INF] Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:09.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:08.416100+0000 mon.smithi151 (mon.0) 1016 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:09.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:08.419206+0000 mon.smithi151 (mon.0) 1017 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:09.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:08.422208+0000 mon.smithi151 (mon.0) 1018 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:09.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:08.426086+0000 mon.smithi151 (mon.0) 1019 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:09.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:08.432193+0000 mon.smithi151 (mon.0) 1020 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:09.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:08.440850+0000 mon.smithi151 (mon.0) 1021 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:09.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:09 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:08.539518+0000 mon.smithi151 (mon.0) 1022 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:09.955 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:39:09.956 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:39:09.956 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:39:09.956 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:39:09.956 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:39:09.956 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:39:09.956 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:39:09.956 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "2/23 daemons upgraded", 2023-11-16T13:39:09.956 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading mgr daemons", 2023-11-16T13:39:09.956 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:39:09.956 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:39:10.528 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:08.665054+0000 mgr.smithi151.fcujra (mgr.14842) 17 : audit [DBG] from='client.14872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:10.528 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:08.976741+0000 mgr.smithi151.fcujra (mgr.14842) 18 : audit [DBG] from='client.14876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:10.528 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:09.275001+0000 mgr.smithi151.fcujra (mgr.14842) 19 : audit [DBG] from='client.14880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:10.528 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:09.557228+0000 mon.smithi151 (mon.0) 1023 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi167.xxnzsz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:39:10.528 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:09.557980+0000 mon.smithi151 (mon.0) 1024 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:39:10.528 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:09.558601+0000 mon.smithi151 (mon.0) 1025 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:10.528 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:09.651846+0000 mon.smithi151 (mon.0) 1026 : audit [DBG] from='client.? 172.21.15.151:0/812492247' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:39:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:08.665054+0000 mgr.smithi151.fcujra (mgr.14842) 17 : audit [DBG] from='client.14872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:08.976741+0000 mgr.smithi151.fcujra (mgr.14842) 18 : audit [DBG] from='client.14876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:09.275001+0000 mgr.smithi151.fcujra (mgr.14842) 19 : audit [DBG] from='client.14880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:09.557228+0000 mon.smithi151 (mon.0) 1023 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi167.xxnzsz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:39:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:09.557980+0000 mon.smithi151 (mon.0) 1024 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:39:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:09.558601+0000 mon.smithi151 (mon.0) 1025 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:10 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:09.651846+0000 mon.smithi151 (mon.0) 1026 : audit [DBG] from='client.? 172.21.15.151:0/812492247' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:39:11.578 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:39:11.678 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:09.556799+0000 mgr.smithi151.fcujra (mgr.14842) 20 : cephadm [INF] Upgrade: Updating mgr.smithi167.xxnzsz 2023-11-16T13:39:11.678 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:09.559246+0000 mgr.smithi151.fcujra (mgr.14842) 21 : cephadm [INF] Deploying daemon mgr.smithi167.xxnzsz on smithi167 2023-11-16T13:39:11.678 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:09.954193+0000 mgr.smithi151.fcujra (mgr.14842) 22 : audit [DBG] from='client.14888 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:11.678 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:10.095626+0000 mgr.smithi151.fcujra (mgr.14842) 23 : cluster [DBG] pgmap v8: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 113 B/s rd, 113 B/s wr, 0 op/s 2023-11-16T13:39:11.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:09.556799+0000 mgr.smithi151.fcujra (mgr.14842) 20 : cephadm [INF] Upgrade: Updating mgr.smithi167.xxnzsz 2023-11-16T13:39:11.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:09.559246+0000 mgr.smithi151.fcujra (mgr.14842) 21 : cephadm [INF] Deploying daemon mgr.smithi167.xxnzsz on smithi167 2023-11-16T13:39:11.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:09.954193+0000 mgr.smithi151.fcujra (mgr.14842) 22 : audit [DBG] from='client.14888 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:11.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:11 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:10.095626+0000 mgr.smithi151.fcujra (mgr.14842) 23 : cluster [DBG] pgmap v8: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 113 B/s rd, 113 B/s wr, 0 op/s 2023-11-16T13:39:12.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:11.589728+0000 mon.smithi151 (mon.0) 1027 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:12.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:11.595801+0000 mon.smithi151 (mon.0) 1028 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:12.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:11.627380+0000 mon.smithi151 (mon.0) 1029 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:12.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:12.096406+0000 mgr.smithi151.fcujra (mgr.14842) 24 : cluster [DBG] pgmap v9: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 186 B/s rd, 186 B/s wr, 0 op/s 2023-11-16T13:39:12.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:11.589728+0000 mon.smithi151 (mon.0) 1027 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:12.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:11.595801+0000 mon.smithi151 (mon.0) 1028 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:12.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:11.627380+0000 mon.smithi151 (mon.0) 1029 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:12.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:12 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:12.096406+0000 mgr.smithi151.fcujra (mgr.14842) 24 : cluster [DBG] pgmap v9: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 186 B/s rd, 186 B/s wr, 0 op/s 2023-11-16T13:39:15.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:14.096909+0000 mgr.smithi151.fcujra (mgr.14842) 25 : cluster [DBG] pgmap v10: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:15.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:14.455168+0000 mon.smithi151 (mon.0) 1030 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:15.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:14.459736+0000 mon.smithi151 (mon.0) 1031 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:15.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:14.096909+0000 mgr.smithi151.fcujra (mgr.14842) 25 : cluster [DBG] pgmap v10: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:15.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:14.455168+0000 mon.smithi151 (mon.0) 1030 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:15.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:15 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:14.459736+0000 mon.smithi151 (mon.0) 1031 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:16.209 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:15.174754+0000 mon.smithi151 (mon.0) 1032 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:39:16.209 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:15.199334+0000 mon.smithi151 (mon.0) 1033 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:16.210 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:15.207458+0000 mon.smithi151 (mon.0) 1034 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:16.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:15.174754+0000 mon.smithi151 (mon.0) 1032 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:39:16.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:15.199334+0000 mon.smithi151 (mon.0) 1033 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:16.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:15.207458+0000 mon.smithi151 (mon.0) 1034 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:16.623 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:39:17.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:16.097430+0000 mgr.smithi151.fcujra (mgr.14842) 26 : cluster [DBG] pgmap v11: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:17.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:17 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:16.097430+0000 mgr.smithi151.fcujra (mgr.14842) 26 : cluster [DBG] pgmap v11: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:18.630 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.534315+0000 mon.smithi151 (mon.0) 1035 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:18.630 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.541021+0000 mon.smithi151 (mon.0) 1036 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:18.631 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.541924+0000 mon.smithi151 (mon.0) 1037 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:18.631 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.542725+0000 mon.smithi151 (mon.0) 1038 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:39:18.631 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.549697+0000 mon.smithi151 (mon.0) 1039 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:18.631 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.558298+0000 mon.smithi151 (mon.0) 1040 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:18.632 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.689292+0000 mon.smithi151 (mon.0) 1041 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:18.632 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.691856+0000 mon.smithi151 (mon.0) 1042 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:39:18.632 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:17.692585+0000 mgr.smithi151.fcujra (mgr.14842) 27 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-11-16T13:39:18.632 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.699127+0000 mon.smithi151 (mon.0) 1043 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:18.632 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.699867+0000 mon.smithi151 (mon.0) 1044 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi151.fcujra"}]: dispatch 2023-11-16T13:39:18.633 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.705126+0000 mon.smithi151 (mon.0) 1045 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi151.fcujra"}]': finished 2023-11-16T13:39:18.633 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.705743+0000 mon.smithi151 (mon.0) 1046 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi167.xxnzsz"}]: dispatch 2023-11-16T13:39:18.633 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.710959+0000 mon.smithi151 (mon.0) 1047 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi167.xxnzsz"}]': finished 2023-11-16T13:39:18.633 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:17.711900+0000 mon.smithi151 (mon.0) 1048 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-11-16T13:39:18.633 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:18.098047+0000 mgr.smithi151.fcujra (mgr.14842) 28 : cluster [DBG] pgmap v12: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:39:18.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.534315+0000 mon.smithi151 (mon.0) 1035 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:18.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.541021+0000 mon.smithi151 (mon.0) 1036 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:18.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.541924+0000 mon.smithi151 (mon.0) 1037 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:18.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.542725+0000 mon.smithi151 (mon.0) 1038 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:39:18.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.549697+0000 mon.smithi151 (mon.0) 1039 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:18.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.558298+0000 mon.smithi151 (mon.0) 1040 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:18.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.689292+0000 mon.smithi151 (mon.0) 1041 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.691856+0000 mon.smithi151 (mon.0) 1042 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:39:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:17.692585+0000 mgr.smithi151.fcujra (mgr.14842) 27 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-11-16T13:39:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.699127+0000 mon.smithi151 (mon.0) 1043 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.699867+0000 mon.smithi151 (mon.0) 1044 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi151.fcujra"}]: dispatch 2023-11-16T13:39:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.705126+0000 mon.smithi151 (mon.0) 1045 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi151.fcujra"}]': finished 2023-11-16T13:39:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.705743+0000 mon.smithi151 (mon.0) 1046 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi167.xxnzsz"}]: dispatch 2023-11-16T13:39:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.710959+0000 mon.smithi151 (mon.0) 1047 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi167.xxnzsz"}]': finished 2023-11-16T13:39:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:17.711900+0000 mon.smithi151 (mon.0) 1048 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-11-16T13:39:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:18.098047+0000 mgr.smithi151.fcujra (mgr.14842) 28 : cluster [DBG] pgmap v12: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:39:19.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:18.596762+0000 mon.smithi151 (mon.0) 1049 : cluster [DBG] Standby manager daemon smithi167.xxnzsz restarted 2023-11-16T13:39:19.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cluster 2023-11-16T13:39:18.596872+0000 mon.smithi151 (mon.0) 1050 : cluster [DBG] Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:19.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:18.597886+0000 mon.smithi167 (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.167:0/3721026821' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/crt"}]: dispatch 2023-11-16T13:39:19.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:18.598381+0000 mon.smithi167 (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.167:0/3721026821' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-16T13:39:19.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:18.600374+0000 mon.smithi167 (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.167:0/3721026821' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/key"}]: dispatch 2023-11-16T13:39:19.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:18.601313+0000 mon.smithi167 (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.167:0/3721026821' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:39:19.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:18.728166+0000 mgr.smithi151.fcujra (mgr.14842) 29 : cephadm [INF] Upgrade: Updating mon.smithi151 2023-11-16T13:39:19.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:18.731728+0000 mon.smithi151 (mon.0) 1051 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:19.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:18.732270+0000 mon.smithi151 (mon.0) 1052 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:39:19.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:18.732700+0000 mon.smithi151 (mon.0) 1053 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-11-16T13:39:19.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: audit 2023-11-16T13:39:18.733162+0000 mon.smithi151 (mon.0) 1054 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:19.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: cephadm 2023-11-16T13:39:18.733745+0000 mgr.smithi151.fcujra (mgr.14842) 30 : cephadm [INF] Deploying daemon mon.smithi151 on smithi151 2023-11-16T13:39:19.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:18.596762+0000 mon.smithi151 (mon.0) 1049 : cluster [DBG] Standby manager daemon smithi167.xxnzsz restarted 2023-11-16T13:39:19.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:18.596872+0000 mon.smithi151 (mon.0) 1050 : cluster [DBG] Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:19.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:18.597886+0000 mon.smithi167 (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.167:0/3721026821' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/crt"}]: dispatch 2023-11-16T13:39:19.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:18.598381+0000 mon.smithi167 (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.167:0/3721026821' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-16T13:39:19.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:18.600374+0000 mon.smithi167 (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.167:0/3721026821' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/key"}]: dispatch 2023-11-16T13:39:19.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:18.601313+0000 mon.smithi167 (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.167:0/3721026821' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:39:19.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:18.728166+0000 mgr.smithi151.fcujra (mgr.14842) 29 : cephadm [INF] Upgrade: Updating mon.smithi151 2023-11-16T13:39:19.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:18.731728+0000 mon.smithi151 (mon.0) 1051 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:19.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:18.732270+0000 mon.smithi151 (mon.0) 1052 : audit [INF] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:39:19.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:18.732700+0000 mon.smithi151 (mon.0) 1053 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-11-16T13:39:19.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:18.733162+0000 mon.smithi151 (mon.0) 1054 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:19.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:18.733745+0000 mgr.smithi151.fcujra (mgr.14842) 30 : cephadm [INF] Deploying daemon mon.smithi151 on smithi151 2023-11-16T13:39:20.097 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 systemd[1]: Stopping Ceph mon.smithi151 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:39:20.097 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:39:19.868+0000 7f8358968700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi151 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-11-16T13:39:20.097 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:19 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151[107562]: debug 2023-11-16T13:39:19.868+0000 7f8358968700 -1 mon.smithi151@0(leader) e2 *** Got Signal Terminated *** 2023-11-16T13:39:20.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:20 smithi151 bash[155364]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi151 2023-11-16T13:39:20.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:20 smithi151 systemd[1]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi151.service: Succeeded. 2023-11-16T13:39:20.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:20 smithi151 systemd[1]: Stopped Ceph mon.smithi151 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:39:20.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:20 smithi151 systemd[1]: Starting Ceph mon.smithi151 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:39:21.409 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: set uid:gid to 167:167 (ceph:ceph) 2023-11-16T13:39:21.409 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable), process ceph-mon, pid 7 2023-11-16T13:39:21.410 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: pidfile_write: ignore empty --pid-file 2023-11-16T13:39:21.410 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 bash[155538]: 77b2d923be97c5521fcab78d7ff16ed61dae430559450b43242a3c6dc09a8046 2023-11-16T13:39:21.410 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 systemd[1]: Started Ceph mon.smithi151 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:39:21.410 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: load: jerasure load: lrc 2023-11-16T13:39:21.410 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: RocksDB version: 6.15.5 2023-11-16T13:39:21.410 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-11-16T13:39:21.410 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Compile date Nov 16 2023 2023-11-16T13:39:21.410 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: DB SUMMARY 2023-11-16T13:39:21.410 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: DB Session ID: B7ZAUNQH6LMFF4T04GVR 2023-11-16T13:39:21.410 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: CURRENT file: CURRENT 2023-11-16T13:39:21.410 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: IDENTITY file: IDENTITY 2023-11-16T13:39:21.411 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: MANIFEST file: MANIFEST-000008 size: 579 Bytes 2023-11-16T13:39:21.411 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi151/store.db dir, Total Num: 1, files: 000017.sst 2023-11-16T13:39:21.411 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi151/store.db: 000015.log size: 6483015 ; 2023-11-16T13:39:21.411 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.error_if_exists: 0 2023-11-16T13:39:21.411 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.create_if_missing: 0 2023-11-16T13:39:21.411 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.paranoid_checks: 1 2023-11-16T13:39:21.411 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-11-16T13:39:21.411 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.env: 0x5572303caa80 2023-11-16T13:39:21.411 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.fs: Posix File System 2023-11-16T13:39:21.411 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.info_log: 0x557232796780 2023-11-16T13:39:21.411 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_file_opening_threads: 16 2023-11-16T13:39:21.411 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.statistics: (nil) 2023-11-16T13:39:21.411 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.use_fsync: 0 2023-11-16T13:39:21.412 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_log_file_size: 0 2023-11-16T13:39:21.412 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-11-16T13:39:21.412 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.log_file_time_to_roll: 0 2023-11-16T13:39:21.412 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.keep_log_file_num: 1000 2023-11-16T13:39:21.412 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.recycle_log_file_num: 0 2023-11-16T13:39:21.412 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.allow_fallocate: 1 2023-11-16T13:39:21.412 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.allow_mmap_reads: 0 2023-11-16T13:39:21.412 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.allow_mmap_writes: 0 2023-11-16T13:39:21.412 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.use_direct_reads: 0 2023-11-16T13:39:21.412 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-11-16T13:39:21.412 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.create_missing_column_families: 0 2023-11-16T13:39:21.412 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.db_log_dir: 2023-11-16T13:39:21.412 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi151/store.db 2023-11-16T13:39:21.413 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.table_cache_numshardbits: 6 2023-11-16T13:39:21.413 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.WAL_ttl_seconds: 0 2023-11-16T13:39:21.413 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.WAL_size_limit_MB: 0 2023-11-16T13:39:21.413 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-11-16T13:39:21.413 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-11-16T13:39:21.413 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.is_fd_close_on_exec: 1 2023-11-16T13:39:21.413 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.advise_random_on_open: 1 2023-11-16T13:39:21.413 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.db_write_buffer_size: 0 2023-11-16T13:39:21.413 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.write_buffer_manager: 0x557233506990 2023-11-16T13:39:21.413 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-11-16T13:39:21.413 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-11-16T13:39:21.413 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-11-16T13:39:21.413 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.use_adaptive_mutex: 0 2023-11-16T13:39:21.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.rate_limiter: (nil) 2023-11-16T13:39:21.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-11-16T13:39:21.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.wal_recovery_mode: 2 2023-11-16T13:39:21.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.enable_thread_tracking: 0 2023-11-16T13:39:21.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.enable_pipelined_write: 0 2023-11-16T13:39:21.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.unordered_write: 0 2023-11-16T13:39:21.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-11-16T13:39:21.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-11-16T13:39:21.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-11-16T13:39:21.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-11-16T13:39:21.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.row_cache: None 2023-11-16T13:39:21.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.wal_filter: None 2023-11-16T13:39:21.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-11-16T13:39:21.415 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.allow_ingest_behind: 0 2023-11-16T13:39:21.415 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.preserve_deletes: 0 2023-11-16T13:39:21.415 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.two_write_queues: 0 2023-11-16T13:39:21.415 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.manual_wal_flush: 0 2023-11-16T13:39:21.415 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.atomic_flush: 0 2023-11-16T13:39:21.415 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-11-16T13:39:21.415 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.persist_stats_to_disk: 0 2023-11-16T13:39:21.415 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.write_dbid_to_manifest: 0 2023-11-16T13:39:21.415 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.log_readahead_size: 0 2023-11-16T13:39:21.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-11-16T13:39:21.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.best_efforts_recovery: 0 2023-11-16T13:39:21.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-11-16T13:39:21.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-11-16T13:39:21.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.allow_data_in_errors: 0 2023-11-16T13:39:21.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.db_host_id: __hostname__ 2023-11-16T13:39:21.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_background_jobs: 2 2023-11-16T13:39:21.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_background_compactions: -1 2023-11-16T13:39:21.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_subcompactions: 1 2023-11-16T13:39:21.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-11-16T13:39:21.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-11-16T13:39:21.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.delayed_write_rate : 16777216 2023-11-16T13:39:21.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_total_wal_size: 0 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.stats_dump_period_sec: 600 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.stats_persist_period_sec: 600 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_open_files: -1 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.bytes_per_sync: 0 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.wal_bytes_per_sync: 0 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.strict_bytes_per_sync: 0 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compaction_readahead_size: 0 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_background_flushes: -1 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Compression algorithms supported: 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: kZSTD supported: 0 2023-11-16T13:39:21.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: kXpressCompression supported: 0 2023-11-16T13:39:21.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: kLZ4HCCompression supported: 1 2023-11-16T13:39:21.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: kZlibCompression supported: 1 2023-11-16T13:39:21.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: kSnappyCompression supported: 1 2023-11-16T13:39:21.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: kLZ4Compression supported: 1 2023-11-16T13:39:21.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: kBZip2Compression supported: 0 2023-11-16T13:39:21.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Fast CRC32 supported: Supported on x86 2023-11-16T13:39:21.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi151/store.db/MANIFEST-000008 2023-11-16T13:39:21.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-11-16T13:39:21.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-11-16T13:39:21.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.merge_operator: 2023-11-16T13:39:21.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compaction_filter: None 2023-11-16T13:39:21.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compaction_filter_factory: None 2023-11-16T13:39:21.419 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.sst_partitioner_factory: None 2023-11-16T13:39:21.419 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.memtable_factory: SkipListFactory 2023-11-16T13:39:21.419 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.table_factory: BlockBasedTable 2023-11-16T13:39:21.419 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5572327e6f90) 2023-11-16T13:39:21.419 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: cache_index_and_filter_blocks: 1 2023-11-16T13:39:21.419 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-11-16T13:39:21.419 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-11-16T13:39:21.419 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: pin_top_level_index_and_filter: 1 2023-11-16T13:39:21.419 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: index_type: 0 2023-11-16T13:39:21.419 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: data_block_index_type: 0 2023-11-16T13:39:21.419 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: index_shortening: 1 2023-11-16T13:39:21.419 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: data_block_hash_table_util_ratio: 0.750000 2023-11-16T13:39:21.419 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: hash_index_allow_collision: 1 2023-11-16T13:39:21.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: checksum: 1 2023-11-16T13:39:21.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: no_block_cache: 0 2023-11-16T13:39:21.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: block_cache: 0x557232819090 2023-11-16T13:39:21.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: block_cache_name: BinnedLRUCache 2023-11-16T13:39:21.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: block_cache_options: 2023-11-16T13:39:21.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: capacity : 536870912 2023-11-16T13:39:21.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: num_shard_bits : 4 2023-11-16T13:39:21.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: strict_capacity_limit : 0 2023-11-16T13:39:21.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: high_pri_pool_ratio: 0.000 2023-11-16T13:39:21.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: block_cache_compressed: (nil) 2023-11-16T13:39:21.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: persistent_cache: (nil) 2023-11-16T13:39:21.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: block_size: 4096 2023-11-16T13:39:21.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: block_size_deviation: 10 2023-11-16T13:39:21.421 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: block_restart_interval: 16 2023-11-16T13:39:21.421 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: index_block_restart_interval: 1 2023-11-16T13:39:21.421 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: metadata_block_size: 4096 2023-11-16T13:39:21.421 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: partition_filters: 0 2023-11-16T13:39:21.421 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: use_delta_encoding: 1 2023-11-16T13:39:21.421 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-11-16T13:39:21.421 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: whole_key_filtering: 1 2023-11-16T13:39:21.421 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: verify_compression: 0 2023-11-16T13:39:21.421 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: read_amp_bytes_per_bit: 0 2023-11-16T13:39:21.421 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: format_version: 4 2023-11-16T13:39:21.421 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: enable_index_compression: 1 2023-11-16T13:39:21.421 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: block_align: 0 2023-11-16T13:39:21.421 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.write_buffer_size: 33554432 2023-11-16T13:39:21.422 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_write_buffer_number: 2 2023-11-16T13:39:21.422 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compression: NoCompression 2023-11-16T13:39:21.422 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.bottommost_compression: Disabled 2023-11-16T13:39:21.422 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.prefix_extractor: nullptr 2023-11-16T13:39:21.422 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-11-16T13:39:21.422 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.num_levels: 7 2023-11-16T13:39:21.422 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-11-16T13:39:21.422 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-11-16T13:39:21.422 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-11-16T13:39:21.422 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-11-16T13:39:21.422 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-11-16T13:39:21.422 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-11-16T13:39:21.422 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-11-16T13:39:21.423 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-11-16T13:39:21.423 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-11-16T13:39:21.423 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-11-16T13:39:21.424 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compression_opts.window_bits: -14 2023-11-16T13:39:21.424 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compression_opts.level: 32767 2023-11-16T13:39:21.424 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compression_opts.strategy: 0 2023-11-16T13:39:21.424 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-11-16T13:39:21.424 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-11-16T13:39:21.424 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-11-16T13:39:21.424 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compression_opts.enabled: false 2023-11-16T13:39:21.424 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-11-16T13:39:21.424 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-11-16T13:39:21.424 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-11-16T13:39:21.424 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.target_file_size_base: 67108864 2023-11-16T13:39:21.425 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.target_file_size_multiplier: 1 2023-11-16T13:39:21.425 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-11-16T13:39:21.425 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-11-16T13:39:21.425 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-11-16T13:39:21.425 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-11-16T13:39:21.425 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-11-16T13:39:21.425 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-11-16T13:39:21.425 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-11-16T13:39:21.425 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-11-16T13:39:21.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-11-16T13:39:21.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-11-16T13:39:21.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-11-16T13:39:21.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-11-16T13:39:21.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.arena_block_size: 4194304 2023-11-16T13:39:21.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-11-16T13:39:21.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-11-16T13:39:21.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-11-16T13:39:21.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.disable_auto_compactions: 0 2023-11-16T13:39:21.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-11-16T13:39:21.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-11-16T13:39:21.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-11-16T13:39:21.426 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-11-16T13:39:21.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-11-16T13:39:21.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-11-16T13:39:21.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-11-16T13:39:21.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-11-16T13:39:21.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-11-16T13:39:21.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-11-16T13:39:21.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.table_properties_collectors: 2023-11-16T13:39:21.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.inplace_update_support: 0 2023-11-16T13:39:21.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.inplace_update_num_locks: 10000 2023-11-16T13:39:21.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-11-16T13:39:21.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-11-16T13:39:21.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.memtable_huge_page_size: 0 2023-11-16T13:39:21.427 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.bloom_locality: 0 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.max_successive_merges: 0 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.optimize_filters_for_hits: 0 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.paranoid_file_checks: 0 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.force_consistency_checks: 1 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.report_bg_io_stats: 0 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.ttl: 2592000 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.periodic_compaction_seconds: 0 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.enable_blob_files: false 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.min_blob_size: 0 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.blob_file_size: 268435456 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.blob_compression_type: NoCompression 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.enable_blob_garbage_collection: false 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-11-16T13:39:21.428 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [db/version_set.cc:3457] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-11-16T13:39:21.429 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi151/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 19, last_sequence is 8758, log_number is 15,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-11-16T13:39:21.429 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 15 2023-11-16T13:39:21.429 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [db/version_set.cc:4082] Creating manifest 19 2023-11-16T13:39:21.429 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [db/version_set.cc:3457] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-11-16T13:39:21.429 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: EVENT_LOG_v1 {"time_micros": 1700141961120366, "job": 1, "event": "recovery_started", "wal_files": [15]} 2023-11-16T13:39:21.429 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #15 mode 2 2023-11-16T13:39:21.429 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-11-16T13:39:21.429 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: EVENT_LOG_v1 {"time_micros": 1700141961136501, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 20, "file_size": 6246539, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 6238288, "index_size": 6222, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1093, "raw_key_size": 8895, "raw_average_key_size": 20, "raw_value_size": 6225565, "raw_average_value_size": 14579, "num_data_blocks": 274, "num_entries": 427, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1700141961, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "d9c39abc-a534-460f-97d8-f7bae5fc6db0", "db_session_id": "B7ZAUNQH6LMFF4T04GVR"}} 2023-11-16T13:39:21.429 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [db/version_set.cc:4082] Creating manifest 21 2023-11-16T13:39:21.429 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [db/version_set.cc:3457] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-11-16T13:39:21.429 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: EVENT_LOG_v1 {"time_micros": 1700141961136974, "job": 1, "event": "recovery_finished"} 2023-11-16T13:39:21.429 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-smithi151/store.db/000015.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-11-16T13:39:21.430 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5572327ad180 2023-11-16T13:39:21.430 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: DB pointer 0x557232882000 2023-11-16T13:39:21.430 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-11-16T13:39:21.430 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-11-16T13:39:21.430 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: ** DB Stats ** 2023-11-16T13:39:21.430 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-11-16T13:39:21.430 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-11-16T13:39:21.430 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-11-16T13:39:21.430 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-16T13:39:21.430 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-11-16T13:39:21.430 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-11-16T13:39:21.430 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-16T13:39:21.431 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: 2023-11-16T13:39:21.431 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: ** Compaction Stats [default] ** 2023-11-16T13:39:21.431 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:39:21.431 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:39:21.431 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: L0 1/0 5.96 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 660.5 0.01 0.00 1 0.009 0 0 2023-11-16T13:39:21.431 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: L6 1/0 21.74 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-16T13:39:21.431 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Sum 2/0 27.69 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 660.5 0.01 0.00 1 0.009 0 0 2023-11-16T13:39:21.431 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 660.5 0.01 0.00 1 0.009 0 0 2023-11-16T13:39:21.431 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: 2023-11-16T13:39:21.431 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: ** Compaction Stats [default] ** 2023-11-16T13:39:21.431 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:39:21.431 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:39:21.431 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 660.5 0.01 0.00 1 0.009 0 0 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Flush(GB): cumulative 0.006, interval 0.006 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: AddFile(Keys): cumulative 0, interval 0 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Cumulative compaction: 0.01 GB write, 306.72 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Interval compaction: 0.01 GB write, 306.72 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: ** File Read Latency Histogram By Level [default] ** 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: ** Compaction Stats [default] ** 2023-11-16T13:39:21.432 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:39:21.433 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:39:21.433 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: L0 1/0 5.96 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 660.5 0.01 0.00 1 0.009 0 0 2023-11-16T13:39:21.433 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: L6 1/0 21.74 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-16T13:39:21.433 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Sum 2/0 27.69 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 660.5 0.01 0.00 1 0.009 0 0 2023-11-16T13:39:21.433 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-16T13:39:21.433 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: 2023-11-16T13:39:21.433 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: ** Compaction Stats [default] ** 2023-11-16T13:39:21.433 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:39:21.433 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:39:21.433 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 660.5 0.01 0.00 1 0.009 0 0 2023-11-16T13:39:21.433 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-11-16T13:39:21.433 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Flush(GB): cumulative 0.006, interval 0.000 2023-11-16T13:39:21.433 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-16T13:39:21.434 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-11-16T13:39:21.434 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-11-16T13:39:21.434 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: AddFile(Keys): cumulative 0, interval 0 2023-11-16T13:39:21.434 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Cumulative compaction: 0.01 GB write, 306.03 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:39:21.434 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:39:21.434 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-16T13:39:21.434 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: 2023-11-16T13:39:21.434 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: ** File Read Latency Histogram By Level [default] ** 2023-11-16T13:39:21.434 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: starting mon.smithi151 rank 0 at public addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] at bind addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi151 fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:39:21.434 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: mon.smithi151@-1(???) e2 preinit fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:39:21.434 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: mon.smithi151@-1(???).mds e8 new map 2023-11-16T13:39:21.434 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: mon.smithi151@-1(???).mds e8 print_map 2023-11-16T13:39:21.435 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: e8 2023-11-16T13:39:21.435 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-11-16T13:39:21.435 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-11-16T13:39:21.435 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: legacy client fscid: 1 2023-11-16T13:39:21.435 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: 2023-11-16T13:39:21.435 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Filesystem 'foofs' (1) 2023-11-16T13:39:21.435 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: fs_name foofs 2023-11-16T13:39:21.435 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: epoch 5 2023-11-16T13:39:21.435 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-11-16T13:39:21.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: created 2023-11-16T13:34:23.917998+0000 2023-11-16T13:39:21.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: modified 2023-11-16T13:34:28.949885+0000 2023-11-16T13:39:21.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: tableserver 0 2023-11-16T13:39:21.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: root 0 2023-11-16T13:39:21.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: session_timeout 60 2023-11-16T13:39:21.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: session_autoclose 300 2023-11-16T13:39:21.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: max_file_size 1099511627776 2023-11-16T13:39:21.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: required_client_features {} 2023-11-16T13:39:21.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: last_failure 0 2023-11-16T13:39:21.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: last_failure_osd_epoch 0 2023-11-16T13:39:21.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-11-16T13:39:21.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: max_mds 1 2023-11-16T13:39:21.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: in 0 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: up {0=24275} 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: failed 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: damaged 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: stopped 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: data_pools [3] 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: metadata_pool 2 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: inline_data disabled 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: balancer 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: standby_count_wanted 1 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: [mds.foofs.smithi167.irznoa{0:24275} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.167:6832/2149060410,v1:172.21.15.167:6833/2149060410] compat {c=[1],r=[1],i=[7ff]}] 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: Standby daemons: 2023-11-16T13:39:21.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: 2023-11-16T13:39:21.438 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout: [mds.foofs.smithi151.mdkcfw{-1:14508} state up:standby seq 3 join_fscid=1 addr [v2:172.21.15.151:6834/3865825489,v1:172.21.15.151:6835/3865825489] compat {c=[1],r=[1],i=[7ff]}] 2023-11-16T13:39:21.438 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: mon.smithi151@-1(???).osd e66 crush map has features 3314933000854323200, adjusting msgr requires 2023-11-16T13:39:21.438 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: mon.smithi151@-1(???).osd e66 crush map has features 432629239337189376, adjusting msgr requires 2023-11-16T13:39:21.438 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: mon.smithi151@-1(???).osd e66 crush map has features 432629239337189376, adjusting msgr requires 2023-11-16T13:39:21.438 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: mon.smithi151@-1(???).osd e66 crush map has features 432629239337189376, adjusting msgr requires 2023-11-16T13:39:21.438 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:21 smithi151 ceph-mon[155575]: mon.smithi151@-1(???).paxosservice(auth 1..42) refresh upgraded, format 0 -> 3 2023-11-16T13:39:21.675 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:39:21.712 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:19.874422+0000 mon.smithi167 (mon.1) 29 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:39:21.712 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:19.878310+0000 mon.smithi167 (mon.1) 30 : audit [DBG] from='mgr.14842 172.21.15.151:0/1016393428' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:39:21.712 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:20.098383+0000 mgr.smithi151.fcujra (mgr.14842) 31 : cluster [DBG] pgmap v13: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:21.712 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:21.348742+0000 mon.smithi151 (mon.0) 1 : cluster [INF] mon.smithi151 calling monitor election 2023-11-16T13:39:21.712 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:21.362439+0000 mon.smithi151 (mon.0) 2 : cluster [INF] mon.smithi151 is new leader, mons smithi151,smithi167 in quorum (ranks 0,1) 2023-11-16T13:39:21.712 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:21.366076+0000 mon.smithi151 (mon.0) 3 : cluster [DBG] monmap e2: 2 mons at {smithi151=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],smithi167=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0]} removed_ranks: {} disallowed_leaders: {} 2023-11-16T13:39:21.712 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:21.366556+0000 mon.smithi151 (mon.0) 4 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 1 up:standby 2023-11-16T13:39:21.713 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:21.366598+0000 mon.smithi151 (mon.0) 5 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-11-16T13:39:21.713 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:21.367430+0000 mon.smithi151 (mon.0) 6 : cluster [DBG] mgrmap e33: smithi151.fcujra(active, since 22s), standbys: smithi167.xxnzsz 2023-11-16T13:39:21.713 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:21.376007+0000 mon.smithi151 (mon.0) 7 : cluster [DBG] mgrmap e34: smithi151.fcujra(active, since 22s), standbys: smithi167.xxnzsz 2023-11-16T13:39:21.713 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:21.376029+0000 mon.smithi151 (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-11-16T13:39:21.713 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:21.376089+0000 mon.smithi151 (mon.0) 9 : audit [INF] from='mgr.14842 ' entity='' 2023-11-16T13:39:23.824 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:22.378423+0000 mon.smithi151 (mon.0) 10 : cluster [DBG] mgrmap e35: smithi151.fcujra(active, since 23s), standbys: smithi167.xxnzsz 2023-11-16T13:39:26.718 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:39:28.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:27.865376+0000 mon.smithi151 (mon.0) 11 : cluster [DBG] Standby manager daemon smithi167.xxnzsz restarted 2023-11-16T13:39:28.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:27.865449+0000 mon.smithi151 (mon.0) 12 : cluster [DBG] Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:28.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.866741+0000 mon.smithi167 (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.167:0/2237517731' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/crt"}]: dispatch 2023-11-16T13:39:28.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.867167+0000 mon.smithi167 (mon.1) 32 : audit [DBG] from='mgr.? 172.21.15.167:0/2237517731' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-16T13:39:28.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.868869+0000 mon.smithi167 (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.167:0/2237517731' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/key"}]: dispatch 2023-11-16T13:39:28.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:27 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.869827+0000 mon.smithi167 (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.167:0/2237517731' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:39:29.325 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:27.920485+0000 mon.smithi151 (mon.0) 13 : cluster [DBG] mgrmap e36: smithi151.fcujra(active, since 28s), standbys: smithi167.xxnzsz 2023-11-16T13:39:29.326 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:27.936898+0000 mon.smithi151 (mon.0) 14 : cluster [INF] Active manager daemon smithi151.fcujra restarted 2023-11-16T13:39:29.326 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:27.937208+0000 mon.smithi151 (mon.0) 15 : cluster [INF] Activating manager daemon smithi151.fcujra 2023-11-16T13:39:29.326 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:27.946187+0000 mon.smithi151 (mon.0) 16 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-11-16T13:39:29.326 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:27.946486+0000 mon.smithi151 (mon.0) 17 : cluster [DBG] mgrmap e37: smithi151.fcujra(active, starting, since 0.00935994s), standbys: smithi167.xxnzsz 2023-11-16T13:39:29.326 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.947713+0000 mon.smithi151 (mon.0) 18 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:39:29.326 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.947915+0000 mon.smithi151 (mon.0) 19 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:39:29.326 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.948731+0000 mon.smithi151 (mon.0) 20 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi151.mdkcfw"}]: dispatch 2023-11-16T13:39:29.326 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.949024+0000 mon.smithi151 (mon.0) 21 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:39:29.326 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.949998+0000 mon.smithi151 (mon.0) 22 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi151.fcujra", "id": "smithi151.fcujra"}]: dispatch 2023-11-16T13:39:29.326 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.950252+0000 mon.smithi151 (mon.0) 23 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi167.xxnzsz", "id": "smithi167.xxnzsz"}]: dispatch 2023-11-16T13:39:29.326 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.950457+0000 mon.smithi151 (mon.0) 24 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:39:29.326 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.950783+0000 mon.smithi151 (mon.0) 25 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:39:29.327 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.952847+0000 mon.smithi151 (mon.0) 26 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:39:29.327 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.953134+0000 mon.smithi151 (mon.0) 27 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:39:29.327 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.953413+0000 mon.smithi151 (mon.0) 28 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:39:29.327 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.953631+0000 mon.smithi151 (mon.0) 29 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:39:29.327 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.953809+0000 mon.smithi151 (mon.0) 30 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:39:29.327 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.954014+0000 mon.smithi151 (mon.0) 31 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:39:29.327 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.954198+0000 mon.smithi151 (mon.0) 32 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-16T13:39:29.327 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.954376+0000 mon.smithi151 (mon.0) 33 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-16T13:39:29.327 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.954811+0000 mon.smithi151 (mon.0) 34 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-16T13:39:29.327 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:27.974619+0000 mon.smithi151 (mon.0) 35 : cluster [INF] Manager daemon smithi151.fcujra is now available 2023-11-16T13:39:29.327 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.996300+0000 mon.smithi151 (mon.0) 36 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:29.327 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:27.998999+0000 mon.smithi151 (mon.0) 37 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:29.327 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:28.021494+0000 mon.smithi151 (mon.0) 38 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:39:29.328 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:28.026695+0000 mon.smithi151 (mon.0) 39 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:39:29.328 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:28.033506+0000 mon.smithi151 (mon.0) 40 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:39:29.328 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:28.753804+0000 mon.smithi151 (mon.0) 41 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:30.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:28.631738+0000 mgr.smithi151.fcujra (mgr.34102) 1 : cephadm [INF] [16/Nov/2023:13:39:28] ENGINE Bus STARTING 2023-11-16T13:39:30.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:28.749702+0000 mgr.smithi151.fcujra (mgr.34102) 2 : cephadm [INF] [16/Nov/2023:13:39:28] ENGINE Serving on https://172.21.15.151:7150 2023-11-16T13:39:30.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:28.749818+0000 mgr.smithi151.fcujra (mgr.34102) 3 : cephadm [INF] [16/Nov/2023:13:39:28] ENGINE Bus STARTED 2023-11-16T13:39:30.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:28.948734+0000 mon.smithi151 (mon.0) 42 : cluster [DBG] mgrmap e38: smithi151.fcujra(active, since 1.01161s), standbys: smithi167.xxnzsz 2023-11-16T13:39:30.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:29 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:28.956523+0000 mgr.smithi151.fcujra (mgr.34102) 4 : cluster [DBG] pgmap v3: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:31.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:30 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:29.951650+0000 mgr.smithi151.fcujra (mgr.34102) 5 : cluster [DBG] pgmap v4: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:31.757 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:39:32.227 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:30.951312+0000 mon.smithi151 (mon.0) 43 : cluster [DBG] mgrmap e39: smithi151.fcujra(active, since 3s), standbys: smithi167.xxnzsz 2023-11-16T13:39:32.227 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:30.960696+0000 mon.smithi151 (mon.0) 44 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:32.227 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:30.964609+0000 mon.smithi151 (mon.0) 45 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:32.227 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:31.652068+0000 mon.smithi151 (mon.0) 46 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:32.227 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:31 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:31.657740+0000 mon.smithi151 (mon.0) 47 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:33.552 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:31.951950+0000 mgr.smithi151.fcujra (mgr.34102) 6 : cluster [DBG] pgmap v5: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:33.552 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:32.106766+0000 mon.smithi151 (mon.0) 48 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:33.552 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:32.111111+0000 mon.smithi151 (mon.0) 49 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:33.552 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:32.796695+0000 mon.smithi151 (mon.0) 50 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:33.552 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:33 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:32.802272+0000 mon.smithi151 (mon.0) 51 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:35.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:33.952405+0000 mgr.smithi151.fcujra (mgr.34102) 7 : cluster [DBG] pgmap v6: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:35.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:33.986152+0000 mon.smithi151 (mon.0) 52 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:35.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:33.990793+0000 mon.smithi151 (mon.0) 53 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:35.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:33.992208+0000 mon.smithi151 (mon.0) 54 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:33.993054+0000 mon.smithi151 (mon.0) 55 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:39:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:33.994215+0000 mgr.smithi151.fcujra (mgr.34102) 8 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-11-16T13:39:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:33.994455+0000 mgr.smithi151.fcujra (mgr.34102) 9 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.conf 2023-11-16T13:39:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:34.258428+0000 mgr.smithi151.fcujra (mgr.34102) 10 : cephadm [INF] Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:34 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:34.258905+0000 mgr.smithi151.fcujra (mgr.34102) 11 : cephadm [INF] Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:36.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:34.522793+0000 mgr.smithi151.fcujra (mgr.34102) 12 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:36.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:34.523280+0000 mgr.smithi151.fcujra (mgr.34102) 13 : cephadm [INF] Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:36.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:34.737169+0000 mgr.smithi151.fcujra (mgr.34102) 14 : cephadm [INF] Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:36.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:34.788114+0000 mgr.smithi151.fcujra (mgr.34102) 15 : cephadm [INF] Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:36.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:35.005166+0000 mon.smithi151 (mon.0) 56 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:36.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:35.009536+0000 mon.smithi151 (mon.0) 57 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:36.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:35.055441+0000 mon.smithi151 (mon.0) 58 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:36.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:35.061028+0000 mon.smithi151 (mon.0) 59 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:36.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:35.064841+0000 mon.smithi151 (mon.0) 60 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:36.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:35.070786+0000 mon.smithi151 (mon.0) 61 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:36.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:35.201560+0000 mon.smithi151 (mon.0) 62 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:36.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:35.203689+0000 mon.smithi151 (mon.0) 63 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:39:36.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:36 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:35.204704+0000 mon.smithi151 (mon.0) 64 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-11-16T13:39:36.794 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:39:37.449 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 systemd[1]: Stopping Ceph mon.smithi167 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:39:37.449 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cluster 2023-11-16T13:39:35.953021+0000 mgr.smithi151.fcujra (mgr.34102) 16 : cluster [DBG] pgmap v7: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 292 B/s rd, 146 B/s wr, 0 op/s 2023-11-16T13:39:37.449 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:36.171432+0000 mgr.smithi151.fcujra (mgr.34102) 17 : cephadm [INF] Upgrade: Updating mon.smithi167 2023-11-16T13:39:37.449 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:36.175151+0000 mon.smithi151 (mon.0) 65 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:37.449 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:36.175987+0000 mon.smithi151 (mon.0) 66 : audit [INF] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:39:37.449 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:36.176597+0000 mon.smithi151 (mon.0) 67 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-11-16T13:39:37.450 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: audit 2023-11-16T13:39:36.177273+0000 mon.smithi151 (mon.0) 68 : audit [DBG] from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:37.450 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: cephadm 2023-11-16T13:39:36.178120+0000 mgr.smithi151.fcujra (mgr.34102) 18 : cephadm [INF] Deploying daemon mon.smithi167 on smithi167 2023-11-16T13:39:37.450 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: debug 2023-11-16T13:39:37.253+0000 7f71f8d2d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi167 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-11-16T13:39:37.450 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167[111541]: debug 2023-11-16T13:39:37.253+0000 7f71f8d2d700 -1 mon.smithi167@1(peon) e2 *** Got Signal Terminated *** 2023-11-16T13:39:37.776 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 bash[138746]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon.smithi167 2023-11-16T13:39:38.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 systemd[1]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi167.service: Succeeded. 2023-11-16T13:39:38.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 systemd[1]: Stopped Ceph mon.smithi167 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:39:38.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:37 smithi167 systemd[1]: Starting Ceph mon.smithi167 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:39:38.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: set uid:gid to 167:167 (ceph:ceph) 2023-11-16T13:39:38.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable), process ceph-mon, pid 7 2023-11-16T13:39:38.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: pidfile_write: ignore empty --pid-file 2023-11-16T13:39:38.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: load: jerasure load: lrc 2023-11-16T13:39:38.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: RocksDB version: 6.15.5 2023-11-16T13:39:38.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-11-16T13:39:38.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Compile date Nov 16 2023 2023-11-16T13:39:38.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: DB SUMMARY 2023-11-16T13:39:38.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: DB Session ID: BDAG8QVFDZ1R7HMTTUXN 2023-11-16T13:39:38.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: CURRENT file: CURRENT 2023-11-16T13:39:38.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: IDENTITY file: IDENTITY 2023-11-16T13:39:38.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: MANIFEST file: MANIFEST-000005 size: 690 Bytes 2023-11-16T13:39:38.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi167/store.db dir, Total Num: 1, files: 000017.sst 2023-11-16T13:39:38.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi167/store.db: 000015.log size: 3041773 ; 2023-11-16T13:39:38.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.error_if_exists: 0 2023-11-16T13:39:38.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.create_if_missing: 0 2023-11-16T13:39:38.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.paranoid_checks: 1 2023-11-16T13:39:38.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-11-16T13:39:38.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.env: 0x565366da0a80 2023-11-16T13:39:38.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.fs: Posix File System 2023-11-16T13:39:38.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.info_log: 0x5653695e4780 2023-11-16T13:39:38.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_file_opening_threads: 16 2023-11-16T13:39:38.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.statistics: (nil) 2023-11-16T13:39:38.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.use_fsync: 0 2023-11-16T13:39:38.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_log_file_size: 0 2023-11-16T13:39:38.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-11-16T13:39:38.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.log_file_time_to_roll: 0 2023-11-16T13:39:38.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.keep_log_file_num: 1000 2023-11-16T13:39:38.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.recycle_log_file_num: 0 2023-11-16T13:39:38.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.allow_fallocate: 1 2023-11-16T13:39:38.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.allow_mmap_reads: 0 2023-11-16T13:39:38.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.allow_mmap_writes: 0 2023-11-16T13:39:38.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.use_direct_reads: 0 2023-11-16T13:39:38.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-11-16T13:39:38.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.create_missing_column_families: 0 2023-11-16T13:39:38.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.db_log_dir: 2023-11-16T13:39:38.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi167/store.db 2023-11-16T13:39:38.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.table_cache_numshardbits: 6 2023-11-16T13:39:38.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.WAL_ttl_seconds: 0 2023-11-16T13:39:38.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.WAL_size_limit_MB: 0 2023-11-16T13:39:38.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-11-16T13:39:38.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-11-16T13:39:38.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.is_fd_close_on_exec: 1 2023-11-16T13:39:38.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.advise_random_on_open: 1 2023-11-16T13:39:38.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.db_write_buffer_size: 0 2023-11-16T13:39:38.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.write_buffer_manager: 0x56536a354990 2023-11-16T13:39:38.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-11-16T13:39:38.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-11-16T13:39:38.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-11-16T13:39:38.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.use_adaptive_mutex: 0 2023-11-16T13:39:38.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.rate_limiter: (nil) 2023-11-16T13:39:38.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-11-16T13:39:38.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.wal_recovery_mode: 2 2023-11-16T13:39:38.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.enable_thread_tracking: 0 2023-11-16T13:39:38.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.enable_pipelined_write: 0 2023-11-16T13:39:38.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.unordered_write: 0 2023-11-16T13:39:38.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-11-16T13:39:38.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-11-16T13:39:38.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-11-16T13:39:38.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-11-16T13:39:38.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.row_cache: None 2023-11-16T13:39:38.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.wal_filter: None 2023-11-16T13:39:38.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-11-16T13:39:38.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.allow_ingest_behind: 0 2023-11-16T13:39:38.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.preserve_deletes: 0 2023-11-16T13:39:38.603 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.two_write_queues: 0 2023-11-16T13:39:38.603 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.manual_wal_flush: 0 2023-11-16T13:39:38.603 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.atomic_flush: 0 2023-11-16T13:39:38.603 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-11-16T13:39:38.603 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.persist_stats_to_disk: 0 2023-11-16T13:39:38.603 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.write_dbid_to_manifest: 0 2023-11-16T13:39:38.603 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.log_readahead_size: 0 2023-11-16T13:39:38.603 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-11-16T13:39:38.603 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.best_efforts_recovery: 0 2023-11-16T13:39:38.603 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-11-16T13:39:38.603 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-11-16T13:39:38.603 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.allow_data_in_errors: 0 2023-11-16T13:39:38.604 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.db_host_id: __hostname__ 2023-11-16T13:39:38.604 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_background_jobs: 2 2023-11-16T13:39:38.604 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_background_compactions: -1 2023-11-16T13:39:38.604 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_subcompactions: 1 2023-11-16T13:39:38.604 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-11-16T13:39:38.604 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-11-16T13:39:38.604 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.delayed_write_rate : 16777216 2023-11-16T13:39:38.604 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_total_wal_size: 0 2023-11-16T13:39:38.604 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-11-16T13:39:38.604 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.stats_dump_period_sec: 600 2023-11-16T13:39:38.605 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.stats_persist_period_sec: 600 2023-11-16T13:39:38.605 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-11-16T13:39:38.605 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_open_files: -1 2023-11-16T13:39:38.605 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.bytes_per_sync: 0 2023-11-16T13:39:38.605 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.wal_bytes_per_sync: 0 2023-11-16T13:39:38.605 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 bash[138907]: 1176c78880ed234528d90398bdfc3806f9d0574729ea4b52f7fa0f5e6e0713b6 2023-11-16T13:39:38.605 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.strict_bytes_per_sync: 0 2023-11-16T13:39:38.605 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compaction_readahead_size: 0 2023-11-16T13:39:38.605 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_background_flushes: -1 2023-11-16T13:39:38.605 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Compression algorithms supported: 2023-11-16T13:39:38.605 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-11-16T13:39:38.605 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: kZSTD supported: 0 2023-11-16T13:39:38.605 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: kXpressCompression supported: 0 2023-11-16T13:39:38.606 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: kLZ4HCCompression supported: 1 2023-11-16T13:39:38.606 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: kZlibCompression supported: 1 2023-11-16T13:39:38.606 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: kSnappyCompression supported: 1 2023-11-16T13:39:38.606 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: kLZ4Compression supported: 1 2023-11-16T13:39:38.606 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: kBZip2Compression supported: 0 2023-11-16T13:39:38.606 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Fast CRC32 supported: Supported on x86 2023-11-16T13:39:38.606 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi167/store.db/MANIFEST-000005 2023-11-16T13:39:38.606 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-11-16T13:39:38.606 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-11-16T13:39:38.606 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.merge_operator: 2023-11-16T13:39:38.606 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compaction_filter: None 2023-11-16T13:39:38.607 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compaction_filter_factory: None 2023-11-16T13:39:38.607 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.sst_partitioner_factory: None 2023-11-16T13:39:38.607 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.memtable_factory: SkipListFactory 2023-11-16T13:39:38.607 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.table_factory: BlockBasedTable 2023-11-16T13:39:38.607 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x565369634f90) 2023-11-16T13:39:38.607 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: cache_index_and_filter_blocks: 1 2023-11-16T13:39:38.607 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-11-16T13:39:38.607 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-11-16T13:39:38.607 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: pin_top_level_index_and_filter: 1 2023-11-16T13:39:38.607 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: index_type: 0 2023-11-16T13:39:38.607 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: data_block_index_type: 0 2023-11-16T13:39:38.607 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: index_shortening: 1 2023-11-16T13:39:38.608 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: data_block_hash_table_util_ratio: 0.750000 2023-11-16T13:39:38.608 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: hash_index_allow_collision: 1 2023-11-16T13:39:38.608 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: checksum: 1 2023-11-16T13:39:38.608 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: no_block_cache: 0 2023-11-16T13:39:38.608 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: block_cache: 0x565369667090 2023-11-16T13:39:38.608 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: block_cache_name: BinnedLRUCache 2023-11-16T13:39:38.608 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: block_cache_options: 2023-11-16T13:39:38.608 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: capacity : 536870912 2023-11-16T13:39:38.608 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: num_shard_bits : 4 2023-11-16T13:39:38.608 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: strict_capacity_limit : 0 2023-11-16T13:39:38.608 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: high_pri_pool_ratio: 0.000 2023-11-16T13:39:38.608 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: block_cache_compressed: (nil) 2023-11-16T13:39:38.608 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: persistent_cache: (nil) 2023-11-16T13:39:38.609 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: block_size: 4096 2023-11-16T13:39:38.609 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: block_size_deviation: 10 2023-11-16T13:39:38.609 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: block_restart_interval: 16 2023-11-16T13:39:38.609 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: index_block_restart_interval: 1 2023-11-16T13:39:38.609 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: metadata_block_size: 4096 2023-11-16T13:39:38.609 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: partition_filters: 0 2023-11-16T13:39:38.609 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: use_delta_encoding: 1 2023-11-16T13:39:38.609 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-11-16T13:39:38.609 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: whole_key_filtering: 1 2023-11-16T13:39:38.609 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: verify_compression: 0 2023-11-16T13:39:38.609 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: read_amp_bytes_per_bit: 0 2023-11-16T13:39:38.609 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: format_version: 4 2023-11-16T13:39:38.609 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: enable_index_compression: 1 2023-11-16T13:39:38.610 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: block_align: 0 2023-11-16T13:39:38.610 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.write_buffer_size: 33554432 2023-11-16T13:39:38.610 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_write_buffer_number: 2 2023-11-16T13:39:38.610 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compression: NoCompression 2023-11-16T13:39:38.610 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.bottommost_compression: Disabled 2023-11-16T13:39:38.610 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.prefix_extractor: nullptr 2023-11-16T13:39:38.610 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-11-16T13:39:38.610 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.num_levels: 7 2023-11-16T13:39:38.610 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-11-16T13:39:38.610 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-11-16T13:39:38.610 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-11-16T13:39:38.611 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-11-16T13:39:38.611 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-11-16T13:39:38.611 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-11-16T13:39:38.611 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-11-16T13:39:38.611 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-11-16T13:39:38.611 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-11-16T13:39:38.611 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-11-16T13:39:38.611 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compression_opts.window_bits: -14 2023-11-16T13:39:38.611 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compression_opts.level: 32767 2023-11-16T13:39:38.611 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compression_opts.strategy: 0 2023-11-16T13:39:38.611 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-11-16T13:39:38.611 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compression_opts.enabled: false 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.target_file_size_base: 67108864 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.target_file_size_multiplier: 1 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-11-16T13:39:38.612 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-11-16T13:39:38.613 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-11-16T13:39:38.613 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-11-16T13:39:38.613 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-11-16T13:39:38.613 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-11-16T13:39:38.613 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-11-16T13:39:38.613 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.arena_block_size: 4194304 2023-11-16T13:39:38.613 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-11-16T13:39:38.613 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-11-16T13:39:38.613 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-11-16T13:39:38.613 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.disable_auto_compactions: 0 2023-11-16T13:39:38.613 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-11-16T13:39:38.613 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-11-16T13:39:38.613 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-11-16T13:39:38.614 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-11-16T13:39:38.614 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-11-16T13:39:38.614 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-11-16T13:39:38.614 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-11-16T13:39:38.614 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-11-16T13:39:38.615 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-11-16T13:39:38.615 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-11-16T13:39:38.615 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.table_properties_collectors: 2023-11-16T13:39:38.615 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.inplace_update_support: 0 2023-11-16T13:39:38.615 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.inplace_update_num_locks: 10000 2023-11-16T13:39:38.615 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-11-16T13:39:38.615 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-11-16T13:39:38.616 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.memtable_huge_page_size: 0 2023-11-16T13:39:38.616 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.bloom_locality: 0 2023-11-16T13:39:38.616 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.max_successive_merges: 0 2023-11-16T13:39:38.616 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.optimize_filters_for_hits: 0 2023-11-16T13:39:38.616 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.paranoid_file_checks: 0 2023-11-16T13:39:38.616 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.force_consistency_checks: 1 2023-11-16T13:39:38.616 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.report_bg_io_stats: 0 2023-11-16T13:39:38.616 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.ttl: 2592000 2023-11-16T13:39:38.616 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.periodic_compaction_seconds: 0 2023-11-16T13:39:38.616 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.enable_blob_files: false 2023-11-16T13:39:38.616 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.min_blob_size: 0 2023-11-16T13:39:38.616 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.blob_file_size: 268435456 2023-11-16T13:39:38.617 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.blob_compression_type: NoCompression 2023-11-16T13:39:38.617 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.enable_blob_garbage_collection: false 2023-11-16T13:39:38.617 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-11-16T13:39:38.617 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi167/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 19, last_sequence is 10672, log_number is 15,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-11-16T13:39:38.617 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 15 2023-11-16T13:39:38.617 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: [db/version_set.cc:4082] Creating manifest 19 2023-11-16T13:39:38.617 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: EVENT_LOG_v1 {"time_micros": 1700141978202594, "job": 1, "event": "recovery_started", "wal_files": [15]} 2023-11-16T13:39:38.617 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #15 mode 2 2023-11-16T13:39:38.617 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-11-16T13:39:38.617 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 systemd[1]: Started Ceph mon.smithi167 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:39:38.617 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: EVENT_LOG_v1 {"time_micros": 1700141978208157, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 20, "file_size": 1955123, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 1952378, "index_size": 1490, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2669, "raw_average_key_size": 22, "raw_value_size": 1948784, "raw_average_value_size": 16239, "num_data_blocks": 67, "num_entries": 120, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1700141978, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "b0751647-14e6-4282-a7b5-356e88bcd73c", "db_session_id": "BDAG8QVFDZ1R7HMTTUXN"}} 2023-11-16T13:39:38.617 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: [db/version_set.cc:4082] Creating manifest 21 2023-11-16T13:39:38.618 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: EVENT_LOG_v1 {"time_micros": 1700141978210632, "job": 1, "event": "recovery_finished"} 2023-11-16T13:39:38.618 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-smithi167/store.db/000015.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-11-16T13:39:38.618 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5653695fb180 2023-11-16T13:39:38.618 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: DB pointer 0x5653696d0000 2023-11-16T13:39:38.618 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-11-16T13:39:38.618 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-11-16T13:39:38.618 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: ** DB Stats ** 2023-11-16T13:39:38.618 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-11-16T13:39:38.618 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-11-16T13:39:38.618 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-11-16T13:39:38.618 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-16T13:39:38.618 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-11-16T13:39:38.618 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: ** Compaction Stats [default] ** 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: L0 1/0 1.86 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 666.4 0.00 0.00 1 0.003 0 0 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: L6 1/0 27.00 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Sum 2/0 28.87 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 666.4 0.00 0.00 1 0.003 0 0 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 666.4 0.00 0.00 1 0.003 0 0 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: ** Compaction Stats [default] ** 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:39:38.619 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 666.4 0.00 0.00 1 0.003 0 0 2023-11-16T13:39:38.620 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-11-16T13:39:38.620 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-11-16T13:39:38.620 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-16T13:39:38.620 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-11-16T13:39:38.620 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-11-16T13:39:38.620 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: AddFile(Keys): cumulative 0, interval 0 2023-11-16T13:39:38.620 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Cumulative compaction: 0.00 GB write, 179.39 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:39:38.620 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Interval compaction: 0.00 GB write, 179.39 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:39:38.620 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-16T13:39:38.620 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: 2023-11-16T13:39:38.620 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: ** File Read Latency Histogram By Level [default] ** 2023-11-16T13:39:38.620 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: 2023-11-16T13:39:38.620 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: ** Compaction Stats [default] ** 2023-11-16T13:39:38.621 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:39:38.621 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:39:38.621 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: L0 1/0 1.86 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 666.4 0.00 0.00 1 0.003 0 0 2023-11-16T13:39:38.621 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: L6 1/0 27.00 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-16T13:39:38.621 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Sum 2/0 28.87 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 666.4 0.00 0.00 1 0.003 0 0 2023-11-16T13:39:38.621 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-16T13:39:38.621 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: 2023-11-16T13:39:38.621 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: ** Compaction Stats [default] ** 2023-11-16T13:39:38.621 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-16T13:39:38.621 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-16T13:39:38.621 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 666.4 0.00 0.00 1 0.003 0 0 2023-11-16T13:39:38.622 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-11-16T13:39:38.622 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Flush(GB): cumulative 0.002, interval 0.000 2023-11-16T13:39:38.622 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-16T13:39:38.622 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-11-16T13:39:38.622 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-11-16T13:39:38.622 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: AddFile(Keys): cumulative 0, interval 0 2023-11-16T13:39:38.622 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Cumulative compaction: 0.00 GB write, 178.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:39:38.622 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-16T13:39:38.622 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-16T13:39:38.622 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: 2023-11-16T13:39:38.622 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: ** File Read Latency Histogram By Level [default] ** 2023-11-16T13:39:38.622 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: starting mon.smithi167 rank 1 at public addrs [v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0] at bind addrs [v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi167 fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:39:38.623 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: mon.smithi167@-1(???) e2 preinit fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:39:38.623 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: mon.smithi167@-1(???).mds e8 new map 2023-11-16T13:39:38.623 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: mon.smithi167@-1(???).mds e8 print_map 2023-11-16T13:39:38.623 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: e8 2023-11-16T13:39:38.623 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-11-16T13:39:38.623 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-11-16T13:39:38.623 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: legacy client fscid: 1 2023-11-16T13:39:38.623 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: 2023-11-16T13:39:38.623 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Filesystem 'foofs' (1) 2023-11-16T13:39:38.623 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: fs_name foofs 2023-11-16T13:39:38.623 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: epoch 5 2023-11-16T13:39:38.623 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-11-16T13:39:38.624 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: created 2023-11-16T13:34:23.917998+0000 2023-11-16T13:39:38.624 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: modified 2023-11-16T13:34:28.949885+0000 2023-11-16T13:39:38.624 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: tableserver 0 2023-11-16T13:39:38.624 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: root 0 2023-11-16T13:39:38.624 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: session_timeout 60 2023-11-16T13:39:38.624 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: session_autoclose 300 2023-11-16T13:39:38.624 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: max_file_size 1099511627776 2023-11-16T13:39:38.624 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: required_client_features {} 2023-11-16T13:39:38.624 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: last_failure 0 2023-11-16T13:39:38.624 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: last_failure_osd_epoch 0 2023-11-16T13:39:38.624 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-11-16T13:39:38.624 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: max_mds 1 2023-11-16T13:39:38.624 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: in 0 2023-11-16T13:39:38.625 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: up {0=24275} 2023-11-16T13:39:38.625 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: failed 2023-11-16T13:39:38.625 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: damaged 2023-11-16T13:39:38.625 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: stopped 2023-11-16T13:39:38.625 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: data_pools [3] 2023-11-16T13:39:38.625 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: metadata_pool 2 2023-11-16T13:39:38.625 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: inline_data disabled 2023-11-16T13:39:38.625 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: balancer 2023-11-16T13:39:38.625 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: standby_count_wanted 1 2023-11-16T13:39:38.625 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: [mds.foofs.smithi167.irznoa{0:24275} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.167:6832/2149060410,v1:172.21.15.167:6833/2149060410] compat {c=[1],r=[1],i=[7ff]}] 2023-11-16T13:39:38.625 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: 2023-11-16T13:39:38.625 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: 2023-11-16T13:39:38.626 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: Standby daemons: 2023-11-16T13:39:38.626 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: 2023-11-16T13:39:38.626 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout: [mds.foofs.smithi151.mdkcfw{-1:14508} state up:standby seq 3 join_fscid=1 addr [v2:172.21.15.151:6834/3865825489,v1:172.21.15.151:6835/3865825489] compat {c=[1],r=[1],i=[7ff]}] 2023-11-16T13:39:38.626 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: mon.smithi167@-1(???).osd e67 crush map has features 3314933000854323200, adjusting msgr requires 2023-11-16T13:39:38.626 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: mon.smithi167@-1(???).osd e67 crush map has features 432629239337189376, adjusting msgr requires 2023-11-16T13:39:38.626 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: mon.smithi167@-1(???).osd e67 crush map has features 432629239337189376, adjusting msgr requires 2023-11-16T13:39:38.626 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: mon.smithi167@-1(???).osd e67 crush map has features 432629239337189376, adjusting msgr requires 2023-11-16T13:39:38.626 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:38 smithi167 ceph-mon[138945]: mon.smithi167@-1(???).paxosservice(auth 1..43) refresh upgraded, format 0 -> 3 2023-11-16T13:39:40.069 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mkfs 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:39:40.070 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Creating key for client.nfs.foo.0.1.smithi151.qfkjic 2023-11-16T13:39:40.070 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Ensuring nfs.foo.0 is in the ganesha grace table 2023-11-16T13:39:40.070 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Rados config object exists: conf-nfs.foo 2023-11-16T13:39:40.071 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Creating key for client.nfs.foo.0.1.smithi151.qfkjic-rgw 2023-11-16T13:39:40.071 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Bind address in nfs.foo.0.1.smithi151.qfkjic's ganesha conf is defaulting to empty 2023-11-16T13:39:40.071 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Deploying daemon nfs.foo.0.1.smithi151.qfkjic on smithi151 2023-11-16T13:39:40.071 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Creating key for client.nfs.foo.1.1.smithi167.rpoanb 2023-11-16T13:39:40.072 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Ensuring nfs.foo.1 is in the ganesha grace table 2023-11-16T13:39:40.072 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Rados config object exists: conf-nfs.foo 2023-11-16T13:39:40.072 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Creating key for client.nfs.foo.1.1.smithi167.rpoanb-rgw 2023-11-16T13:39:40.073 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Bind address in nfs.foo.1.1.smithi167.rpoanb's ganesha conf is defaulting to empty 2023-11-16T13:39:40.073 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Deploying daemon nfs.foo.1.1.smithi167.rpoanb on smithi167 2023-11-16T13:39:40.073 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Reconfiguring prometheus.smithi151 (dependencies changed)... 2023-11-16T13:39:40.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Reconfiguring daemon prometheus.smithi151 on smithi151 2023-11-16T13:39:40.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:39:40.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:39:40.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Upgrade: Updating mgr.smithi151.fcujra 2023-11-16T13:39:40.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Deploying daemon mgr.smithi151.fcujra on smithi151 2023-11-16T13:39:40.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v92: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:40.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Standby manager daemon smithi151.fcujra restarted 2023-11-16T13:39:40.074 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Standby manager daemon smithi151.fcujra started 2023-11-16T13:39:40.075 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mgrmap e29: smithi167.xxnzsz(active, since 2m), standbys: smithi151.fcujra 2023-11-16T13:39:40.075 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:39:40.075 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:39:40.075 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Failing over to other MGR 2023-11-16T13:39:40.075 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: osdmap e66: 8 total, 8 up, 8 in 2023-11-16T13:39:40.075 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v94: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:39:40.075 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:40.075 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mgrmap e30: smithi151.fcujra(active, starting, since 0.8731s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.076 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Manager daemon smithi151.fcujra is now available 2023-11-16T13:39:40.076 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: [16/Nov/2023:13:39:00] ENGINE Bus STARTING 2023-11-16T13:39:40.076 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: [16/Nov/2023:13:39:00] ENGINE Serving on https://172.21.15.151:7150 2023-11-16T13:39:40.076 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: [16/Nov/2023:13:39:00] ENGINE Bus STARTED 2023-11-16T13:39:40.076 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mgrmap e31: smithi151.fcujra(active, since 1.87472s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.076 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v3: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.076 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v4: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.076 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v5: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.076 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v6: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.076 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Standby manager daemon smithi167.xxnzsz restarted 2023-11-16T13:39:40.076 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:40.077 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi151:/etc/ceph/ceph.conf 2023-11-16T13:39:40.077 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi167:/etc/ceph/ceph.conf 2023-11-16T13:39:40.077 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mgrmap e32: smithi151.fcujra(active, since 8s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.077 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:40.077 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:40.077 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:40.077 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:40.077 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v7: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 146 B/s rd, 146 B/s wr, 0 op/s 2023-11-16T13:39:40.077 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:40.077 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:40.077 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Upgrade: Updating mgr.smithi167.xxnzsz 2023-11-16T13:39:40.077 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Deploying daemon mgr.smithi167.xxnzsz on smithi167 2023-11-16T13:39:40.077 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v8: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 113 B/s rd, 113 B/s wr, 0 op/s 2023-11-16T13:39:40.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v9: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 186 B/s rd, 186 B/s wr, 0 op/s 2023-11-16T13:39:40.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v10: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:40.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v11: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:40.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Upgrade: Setting container_image for all mgr 2023-11-16T13:39:40.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v12: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:39:40.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Standby manager daemon smithi167.xxnzsz restarted 2023-11-16T13:39:40.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:40.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Upgrade: Updating mon.smithi151 2023-11-16T13:39:40.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Deploying daemon mon.smithi151 on smithi151 2023-11-16T13:39:40.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v13: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:40.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mon.smithi151 calling monitor election 2023-11-16T13:39:40.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mon.smithi151 is new leader, mons smithi151,smithi167 in quorum (ranks 0,1) 2023-11-16T13:39:40.079 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: monmap e2: 2 mons at {smithi151=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],smithi167=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0]} removed_ranks: {} disallowed_leaders: {} 2023-11-16T13:39:40.079 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 1 up:standby 2023-11-16T13:39:40.079 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: osdmap e66: 8 total, 8 up, 8 in 2023-11-16T13:39:40.079 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mgrmap e33: smithi151.fcujra(active, since 22s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.079 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mgrmap e34: smithi151.fcujra(active, since 22s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.080 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: overall HEALTH_OK 2023-11-16T13:39:40.080 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mgrmap e35: smithi151.fcujra(active, since 23s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.080 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Standby manager daemon smithi167.xxnzsz restarted 2023-11-16T13:39:40.080 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:40.080 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.? 172.21.15.167:0/2237517731' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/key"}]: dispatch 2023-11-16T13:39:40.080 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.? 172.21.15.167:0/2237517731' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:39:40.081 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mgrmap e36: smithi151.fcujra(active, since 28s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.081 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Active manager daemon smithi151.fcujra restarted 2023-11-16T13:39:40.081 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Activating manager daemon smithi151.fcujra 2023-11-16T13:39:40.081 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: osdmap e67: 8 total, 8 up, 8 in 2023-11-16T13:39:40.081 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mgrmap e37: smithi151.fcujra(active, starting, since 0.00935994s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.081 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:39:40.082 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:39:40.082 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi151.mdkcfw"}]: dispatch 2023-11-16T13:39:40.082 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:39:40.082 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi151.fcujra", "id": "smithi151.fcujra"}]: dispatch 2023-11-16T13:39:40.082 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi167.xxnzsz", "id": "smithi167.xxnzsz"}]: dispatch 2023-11-16T13:39:40.082 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:39:40.082 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:39:40.082 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:39:40.083 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:39:40.083 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:39:40.083 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:39:40.083 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:39:40.083 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:39:40.083 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-16T13:39:40.084 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-16T13:39:40.084 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-16T13:39:40.084 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Manager daemon smithi151.fcujra is now available 2023-11-16T13:39:40.084 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:40.084 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:40.084 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:39:40.085 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:39:40.085 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:39:40.085 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: [16/Nov/2023:13:39:28] ENGINE Bus STARTING 2023-11-16T13:39:40.085 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: [16/Nov/2023:13:39:28] ENGINE Serving on https://172.21.15.151:7150 2023-11-16T13:39:40.085 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: [16/Nov/2023:13:39:28] ENGINE Bus STARTED 2023-11-16T13:39:40.085 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.086 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mgrmap e38: smithi151.fcujra(active, since 1.01161s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.086 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v3: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.086 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v4: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.086 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mgrmap e39: smithi151.fcujra(active, since 3s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.086 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.086 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.087 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.087 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.087 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v5: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.087 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.087 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.087 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.088 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.088 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v6: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.088 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.088 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.088 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:40.088 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:39:40.089 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi151:/etc/ceph/ceph.conf 2023-11-16T13:39:40.089 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi167:/etc/ceph/ceph.conf 2023-11-16T13:39:40.089 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:40.089 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:40.089 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:40.090 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:40.090 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:40.090 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:40.090 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.090 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.090 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.091 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.091 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.091 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.091 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:40.091 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:39:40.091 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-11-16T13:39:40.092 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: pgmap v7: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 292 B/s rd, 146 B/s wr, 0 op/s 2023-11-16T13:39:40.092 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Upgrade: Updating mon.smithi167 2023-11-16T13:39:40.092 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.092 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:39:40.092 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-11-16T13:39:40.093 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:40.093 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: Deploying daemon mon.smithi167 on smithi167 2023-11-16T13:39:40.093 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: overall HEALTH_OK 2023-11-16T13:39:40.093 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:39:40.093 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mon.smithi167 calling monitor election 2023-11-16T13:39:40.093 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mon.smithi151 calling monitor election 2023-11-16T13:39:40.093 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:39:40.093 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mon.smithi151 is new leader, mons smithi151,smithi167 in quorum (ranks 0,1) 2023-11-16T13:39:40.093 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: monmap e3: 2 mons at {smithi151=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],smithi167=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0]} removed_ranks: {} disallowed_leaders: {} 2023-11-16T13:39:40.093 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 1 up:standby 2023-11-16T13:39:40.093 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: osdmap e67: 8 total, 8 up, 8 in 2023-11-16T13:39:40.094 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: mgrmap e39: smithi151.fcujra(active, since 10s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.094 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: overall HEALTH_OK 2023-11-16T13:39:40.094 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.094 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.094 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:40.098 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mkfs 11fdb452-8484-11ee-8dbe-212e2dc638e7 2023-11-16T13:39:40.099 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Creating key for client.nfs.foo.0.1.smithi151.qfkjic 2023-11-16T13:39:40.099 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Ensuring nfs.foo.0 is in the ganesha grace table 2023-11-16T13:39:40.099 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Rados config object exists: conf-nfs.foo 2023-11-16T13:39:40.099 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Creating key for client.nfs.foo.0.1.smithi151.qfkjic-rgw 2023-11-16T13:39:40.099 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Bind address in nfs.foo.0.1.smithi151.qfkjic's ganesha conf is defaulting to empty 2023-11-16T13:39:40.100 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Deploying daemon nfs.foo.0.1.smithi151.qfkjic on smithi151 2023-11-16T13:39:40.100 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Creating key for client.nfs.foo.1.1.smithi167.rpoanb 2023-11-16T13:39:40.100 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Ensuring nfs.foo.1 is in the ganesha grace table 2023-11-16T13:39:40.100 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Rados config object exists: conf-nfs.foo 2023-11-16T13:39:40.100 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Creating key for client.nfs.foo.1.1.smithi167.rpoanb-rgw 2023-11-16T13:39:40.100 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Bind address in nfs.foo.1.1.smithi167.rpoanb's ganesha conf is defaulting to empty 2023-11-16T13:39:40.101 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Deploying daemon nfs.foo.1.1.smithi167.rpoanb on smithi167 2023-11-16T13:39:40.101 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Reconfiguring prometheus.smithi151 (dependencies changed)... 2023-11-16T13:39:40.101 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Reconfiguring daemon prometheus.smithi151 on smithi151 2023-11-16T13:39:40.101 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:39:40.101 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:39:40.101 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Upgrade: Updating mgr.smithi151.fcujra 2023-11-16T13:39:40.101 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Deploying daemon mgr.smithi151.fcujra on smithi151 2023-11-16T13:39:40.102 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v92: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:40.102 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Standby manager daemon smithi151.fcujra restarted 2023-11-16T13:39:40.102 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Standby manager daemon smithi151.fcujra started 2023-11-16T13:39:40.102 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mgrmap e29: smithi167.xxnzsz(active, since 2m), standbys: smithi151.fcujra 2023-11-16T13:39:40.102 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:39:40.102 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Upgrade: Need to upgrade myself (mgr.smithi167.xxnzsz) 2023-11-16T13:39:40.102 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Failing over to other MGR 2023-11-16T13:39:40.103 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: osdmap e66: 8 total, 8 up, 8 in 2023-11-16T13:39:40.103 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v94: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:39:40.103 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:40.103 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mgrmap e30: smithi151.fcujra(active, starting, since 0.8731s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.103 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Manager daemon smithi151.fcujra is now available 2023-11-16T13:39:40.103 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: [16/Nov/2023:13:39:00] ENGINE Bus STARTING 2023-11-16T13:39:40.104 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: [16/Nov/2023:13:39:00] ENGINE Serving on https://172.21.15.151:7150 2023-11-16T13:39:40.104 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: [16/Nov/2023:13:39:00] ENGINE Bus STARTED 2023-11-16T13:39:40.104 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mgrmap e31: smithi151.fcujra(active, since 1.87472s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.104 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v3: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.104 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v4: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.104 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v5: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.104 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v6: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.105 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Standby manager daemon smithi167.xxnzsz restarted 2023-11-16T13:39:40.105 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:40.105 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi151:/etc/ceph/ceph.conf 2023-11-16T13:39:40.105 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi167:/etc/ceph/ceph.conf 2023-11-16T13:39:40.105 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mgrmap e32: smithi151.fcujra(active, since 8s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.105 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:40.106 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:40.106 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:40.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:40.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v7: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 146 B/s rd, 146 B/s wr, 0 op/s 2023-11-16T13:39:40.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:40.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:40.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Upgrade: Updating mgr.smithi167.xxnzsz 2023-11-16T13:39:40.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Deploying daemon mgr.smithi167.xxnzsz on smithi167 2023-11-16T13:39:40.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v8: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 113 B/s rd, 113 B/s wr, 0 op/s 2023-11-16T13:39:40.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v9: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 186 B/s rd, 186 B/s wr, 0 op/s 2023-11-16T13:39:40.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v10: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:40.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v11: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:40.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Upgrade: Setting container_image for all mgr 2023-11-16T13:39:40.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v12: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:39:40.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Standby manager daemon smithi167.xxnzsz restarted 2023-11-16T13:39:40.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:40.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Upgrade: Updating mon.smithi151 2023-11-16T13:39:40.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Deploying daemon mon.smithi151 on smithi151 2023-11-16T13:39:40.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v13: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:40.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mon.smithi151 calling monitor election 2023-11-16T13:39:40.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mon.smithi151 is new leader, mons smithi151,smithi167 in quorum (ranks 0,1) 2023-11-16T13:39:40.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: monmap e2: 2 mons at {smithi151=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],smithi167=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0]} removed_ranks: {} disallowed_leaders: {} 2023-11-16T13:39:40.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 1 up:standby 2023-11-16T13:39:40.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: osdmap e66: 8 total, 8 up, 8 in 2023-11-16T13:39:40.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mgrmap e33: smithi151.fcujra(active, since 22s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.109 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mgrmap e34: smithi151.fcujra(active, since 22s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.109 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: overall HEALTH_OK 2023-11-16T13:39:40.109 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mgrmap e35: smithi151.fcujra(active, since 23s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.109 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Standby manager daemon smithi167.xxnzsz restarted 2023-11-16T13:39:40.109 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Standby manager daemon smithi167.xxnzsz started 2023-11-16T13:39:40.109 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.? 172.21.15.167:0/2237517731' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi167.xxnzsz/key"}]: dispatch 2023-11-16T13:39:40.109 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.? 172.21.15.167:0/2237517731' entity='mgr.smithi167.xxnzsz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-16T13:39:40.109 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mgrmap e36: smithi151.fcujra(active, since 28s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.109 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Active manager daemon smithi151.fcujra restarted 2023-11-16T13:39:40.109 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Activating manager daemon smithi151.fcujra 2023-11-16T13:39:40.109 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: osdmap e67: 8 total, 8 up, 8 in 2023-11-16T13:39:40.109 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mgrmap e37: smithi151.fcujra(active, starting, since 0.00935994s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.109 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:39:40.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:39:40.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi151.mdkcfw"}]: dispatch 2023-11-16T13:39:40.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:39:40.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi151.fcujra", "id": "smithi151.fcujra"}]: dispatch 2023-11-16T13:39:40.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr metadata", "who": "smithi167.xxnzsz", "id": "smithi167.xxnzsz"}]: dispatch 2023-11-16T13:39:40.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:39:40.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:39:40.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:39:40.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:39:40.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:39:40.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:39:40.110 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Manager daemon smithi151.fcujra is now available 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/mirror_snapshot_schedule"}]: dispatch 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi151.fcujra/trash_purge_schedule"}]: dispatch 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: [16/Nov/2023:13:39:28] ENGINE Bus STARTING 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: [16/Nov/2023:13:39:28] ENGINE Serving on https://172.21.15.151:7150 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: [16/Nov/2023:13:39:28] ENGINE Bus STARTED 2023-11-16T13:39:40.111 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.112 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mgrmap e38: smithi151.fcujra(active, since 1.01161s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.112 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v3: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.112 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v4: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.112 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mgrmap e39: smithi151.fcujra(active, since 3s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.112 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.112 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.112 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.112 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.112 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v5: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.112 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.112 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.112 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.113 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.113 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v6: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-11-16T13:39:40.113 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.113 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.113 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:40.113 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:39:40.113 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi151:/etc/ceph/ceph.conf 2023-11-16T13:39:40.113 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi167:/etc/ceph/ceph.conf 2023-11-16T13:39:40.113 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:40.113 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.conf 2023-11-16T13:39:40.113 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:40.113 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi167:/etc/ceph/ceph.client.admin.keyring 2023-11-16T13:39:40.114 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:40.114 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Updating smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/config/ceph.client.admin.keyring 2023-11-16T13:39:40.114 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.114 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.114 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.114 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.114 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.114 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.114 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:40.114 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:39:40.114 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-11-16T13:39:40.114 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: pgmap v7: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 292 B/s rd, 146 B/s wr, 0 op/s 2023-11-16T13:39:40.115 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Upgrade: Updating mon.smithi167 2023-11-16T13:39:40.115 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.115 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:39:40.115 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-11-16T13:39:40.115 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:40.115 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: Deploying daemon mon.smithi167 on smithi167 2023-11-16T13:39:40.115 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: overall HEALTH_OK 2023-11-16T13:39:40.115 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi151"}]: dispatch 2023-11-16T13:39:40.115 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mon.smithi167 calling monitor election 2023-11-16T13:39:40.115 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mon.smithi151 calling monitor election 2023-11-16T13:39:40.115 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mon metadata", "id": "smithi167"}]: dispatch 2023-11-16T13:39:40.115 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mon.smithi151 is new leader, mons smithi151,smithi167 in quorum (ranks 0,1) 2023-11-16T13:39:40.116 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: monmap e3: 2 mons at {smithi151=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],smithi167=[v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0]} removed_ranks: {} disallowed_leaders: {} 2023-11-16T13:39:40.116 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 1 up:standby 2023-11-16T13:39:40.116 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: osdmap e67: 8 total, 8 up, 8 in 2023-11-16T13:39:40.116 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: mgrmap e39: smithi151.fcujra(active, since 10s), standbys: smithi167.xxnzsz 2023-11-16T13:39:40.116 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: overall HEALTH_OK 2023-11-16T13:39:40.116 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.116 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:40.117 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:39:40.292 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:39:40.903 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:39:40.904 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (7m) 8s ago 8m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:39:40.904 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (8m) 8s ago 8m 7008k - 16.2.5 6933c2a0b7dd 954bd32d4ee2 2023-11-16T13:39:40.904 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (8m) 9s ago 8m 7155k - 16.2.5 6933c2a0b7dd 649801b49aea 2023-11-16T13:39:40.904 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (7m) 8s ago 8m 36.0M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:39:40.904 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (5m) 8s ago 5m 15.8M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:39:40.905 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (5m) 9s ago 5m 42.3M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:39:40.905 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (49s) 8s ago 10m 468M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:39:40.905 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (29s) 9s ago 8m 436M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:39:40.905 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (19s) 8s ago 10m 61.8M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:39:40.905 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 starting - - - 2048M 2023-11-16T13:39:40.905 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (3m) 8s ago 3m 81.2M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:39:40.905 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (3m) 9s ago 3m 56.1M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:39:40.906 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (8m) 8s ago 8m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:39:40.906 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (7m) 9s ago 7m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:39:40.906 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (7m) 8s ago 7m 1228M 1944M 16.2.5 6933c2a0b7dd 7598578fb701 2023-11-16T13:39:40.906 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (7m) 8s ago 7m 1079M 1944M 16.2.5 6933c2a0b7dd 989f26649fd3 2023-11-16T13:39:40.906 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (7m) 8s ago 7m 643M 1944M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:39:40.906 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (6m) 8s ago 6m 1158M 1944M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:39:40.906 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (6m) 9s ago 6m 1162M 2712M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:39:40.907 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (6m) 9s ago 6m 1362M 2712M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:39:40.907 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (6m) 9s ago 6m 1048M 2712M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:39:40.907 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (6m) 9s ago 6m 1814M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:39:40.907 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (52s) 8s ago 8m 50.1M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:39:41.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:40 smithi167 ceph-mon[138945]: pgmap v9: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 186 B/s rd, 186 B/s wr, 0 op/s 2023-11-16T13:39:41.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:40 smithi167 ceph-mon[138945]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:41.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:40 smithi151 ceph-mon[155575]: pgmap v9: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 186 B/s rd, 186 B/s wr, 0 op/s 2023-11-16T13:39:41.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:40 smithi151 ceph-mon[155575]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:41.270 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:39:41.271 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:39:41.271 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:39:41.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:39:41.271 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:39:41.271 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:39:41.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:39:41.271 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:39:41.271 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2023-11-16T13:39:41.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:39:41.271 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:39:41.271 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:39:41.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:39:41.272 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:39:41.272 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 10, 2023-11-16T13:39:41.272 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 4 2023-11-16T13:39:41.272 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:39:41.272 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:39:41.577 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:39:41.577 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:39:41.577 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:39:41.577 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:39:41.577 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:39:41.578 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:39:41.578 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:39:41.578 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "3/23 daemons upgraded", 2023-11-16T13:39:41.578 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading mon daemons", 2023-11-16T13:39:41.578 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:39:41.578 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:39:41.833 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:39:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:42 smithi151 ceph-mon[155575]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:42 smithi151 ceph-mon[155575]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:42 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/649639044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:39:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:42.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:42 smithi167 ceph-mon[138945]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:42.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:42 smithi167 ceph-mon[138945]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:42.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:42 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/649639044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:39:42.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:42.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:43.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:43 smithi151 ceph-mon[155575]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:43.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:43 smithi151 ceph-mon[155575]: pgmap v10: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:43.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:43.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:43.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:39:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:43 smithi167 ceph-mon[138945]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:39:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:43 smithi167 ceph-mon[138945]: pgmap v10: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:39:45.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:45 smithi151 ceph-mon[155575]: pgmap v11: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:45.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:45 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:45.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:45 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:45.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:45 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:45.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:45 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:39:45.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:45 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:45.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:45 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:45.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:45 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:39:45.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:45 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-11-16T13:39:45.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:45 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:45 smithi167 ceph-mon[138945]: pgmap v11: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:45 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:45 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:45 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:45 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:39:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:45 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:45 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:45 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:39:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:45 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-11-16T13:39:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:45 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:46.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:46 smithi167 ceph-mon[138945]: Reconfiguring mon.smithi151 (monmap changed)... 2023-11-16T13:39:46.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:46 smithi167 ceph-mon[138945]: Reconfiguring daemon mon.smithi151 on smithi151 2023-11-16T13:39:46.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:46 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:46 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:46 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi151.fcujra", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:39:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:46 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:39:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:46 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:46.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:46 smithi151 ceph-mon[155575]: Reconfiguring mon.smithi151 (monmap changed)... 2023-11-16T13:39:46.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:46 smithi151 ceph-mon[155575]: Reconfiguring daemon mon.smithi151 on smithi151 2023-11-16T13:39:46.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:46 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:46.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:46 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:46.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:46 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi151.fcujra", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:39:46.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:46 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:39:46.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:46 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:46.867 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:39:47.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:47 smithi167 ceph-mon[138945]: Reconfiguring mgr.smithi151.fcujra (monmap changed)... 2023-11-16T13:39:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:47 smithi167 ceph-mon[138945]: Reconfiguring daemon mgr.smithi151.fcujra on smithi151 2023-11-16T13:39:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:47 smithi167 ceph-mon[138945]: pgmap v12: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:39:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi151", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:39:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:47.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:47 smithi151 ceph-mon[155575]: Reconfiguring mgr.smithi151.fcujra (monmap changed)... 2023-11-16T13:39:47.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:47 smithi151 ceph-mon[155575]: Reconfiguring daemon mgr.smithi151.fcujra on smithi151 2023-11-16T13:39:47.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:47 smithi151 ceph-mon[155575]: pgmap v12: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:39:47.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:47.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:47.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi151", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:39:47.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:48.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:48 smithi167 ceph-mon[138945]: Reconfiguring crash.smithi151 (monmap changed)... 2023-11-16T13:39:48.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:48 smithi167 ceph-mon[138945]: Reconfiguring daemon crash.smithi151 on smithi151 2023-11-16T13:39:48.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:48 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:48.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:48 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:48.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:48 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-11-16T13:39:48.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:48 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:48.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:48 smithi151 ceph-mon[155575]: Reconfiguring crash.smithi151 (monmap changed)... 2023-11-16T13:39:48.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:48 smithi151 ceph-mon[155575]: Reconfiguring daemon crash.smithi151 on smithi151 2023-11-16T13:39:48.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:48 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:48.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:48 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:48.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:48 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-11-16T13:39:48.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:48 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:49.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:49 smithi167 ceph-mon[138945]: Reconfiguring osd.0 (monmap changed)... 2023-11-16T13:39:49.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:49 smithi167 ceph-mon[138945]: Reconfiguring daemon osd.0 on smithi151 2023-11-16T13:39:49.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:49 smithi167 ceph-mon[138945]: pgmap v13: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:49.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:49 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:49.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:49 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:49.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:49 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-11-16T13:39:49.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:49 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:49.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:49 smithi151 ceph-mon[155575]: Reconfiguring osd.0 (monmap changed)... 2023-11-16T13:39:49.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:49 smithi151 ceph-mon[155575]: Reconfiguring daemon osd.0 on smithi151 2023-11-16T13:39:49.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:49 smithi151 ceph-mon[155575]: pgmap v13: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:49.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:49 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:49.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:49 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:49.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:49 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-11-16T13:39:49.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:49 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:50.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:50 smithi167 ceph-mon[138945]: Reconfiguring osd.1 (monmap changed)... 2023-11-16T13:39:50.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:50 smithi167 ceph-mon[138945]: Reconfiguring daemon osd.1 on smithi151 2023-11-16T13:39:50.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:50 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:50.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:50 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:50.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:50 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-11-16T13:39:50.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:50 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:50.642 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:50 smithi151 ceph-mon[155575]: Reconfiguring osd.1 (monmap changed)... 2023-11-16T13:39:50.642 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:50 smithi151 ceph-mon[155575]: Reconfiguring daemon osd.1 on smithi151 2023-11-16T13:39:50.642 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:50 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:50.642 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:50 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:50.642 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:50 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-11-16T13:39:50.642 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:50 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:51.438 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:51 smithi151 ceph-mon[155575]: Reconfiguring osd.2 (monmap changed)... 2023-11-16T13:39:51.438 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:51 smithi151 ceph-mon[155575]: Reconfiguring daemon osd.2 on smithi151 2023-11-16T13:39:51.438 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:51 smithi151 ceph-mon[155575]: pgmap v14: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:39:51.438 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:51 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:51.438 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:51 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:51.439 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:51 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-11-16T13:39:51.439 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:51 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:51.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:51 smithi167 ceph-mon[138945]: Reconfiguring osd.2 (monmap changed)... 2023-11-16T13:39:51.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:51 smithi167 ceph-mon[138945]: Reconfiguring daemon osd.2 on smithi151 2023-11-16T13:39:51.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:51 smithi167 ceph-mon[138945]: pgmap v14: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:39:51.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:51 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:51.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:51 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:51.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:51 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-11-16T13:39:51.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:51 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:51.914 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:39:52.463 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:52 smithi151 ceph-mon[155575]: Reconfiguring osd.3 (monmap changed)... 2023-11-16T13:39:52.463 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:52 smithi151 ceph-mon[155575]: Reconfiguring daemon osd.3 on smithi151 2023-11-16T13:39:52.463 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:52 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:52.463 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:52 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:52.463 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:52 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi151.mdkcfw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-11-16T13:39:52.464 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:52 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:52.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:52 smithi167 ceph-mon[138945]: Reconfiguring osd.3 (monmap changed)... 2023-11-16T13:39:52.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:52 smithi167 ceph-mon[138945]: Reconfiguring daemon osd.3 on smithi151 2023-11-16T13:39:52.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:52 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:52.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:52 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:52.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:52 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi151.mdkcfw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-11-16T13:39:52.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:52 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:53.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:53 smithi167 ceph-mon[138945]: Reconfiguring mds.foofs.smithi151.mdkcfw (monmap changed)... 2023-11-16T13:39:53.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:53 smithi167 ceph-mon[138945]: Reconfiguring daemon mds.foofs.smithi151.mdkcfw on smithi151 2023-11-16T13:39:53.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:53 smithi167 ceph-mon[138945]: pgmap v15: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:53.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:53.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:53.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi167", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:39:53.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:53 smithi151 ceph-mon[155575]: Reconfiguring mds.foofs.smithi151.mdkcfw (monmap changed)... 2023-11-16T13:39:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:53 smithi151 ceph-mon[155575]: Reconfiguring daemon mds.foofs.smithi151.mdkcfw on smithi151 2023-11-16T13:39:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:53 smithi151 ceph-mon[155575]: pgmap v15: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi167", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:39:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:54.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:54 smithi167 ceph-mon[138945]: Reconfiguring crash.smithi167 (monmap changed)... 2023-11-16T13:39:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:54 smithi167 ceph-mon[138945]: Reconfiguring daemon crash.smithi167 on smithi167 2023-11-16T13:39:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi167.xxnzsz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:39:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:39:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:54 smithi151 ceph-mon[155575]: Reconfiguring crash.smithi167 (monmap changed)... 2023-11-16T13:39:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:54 smithi151 ceph-mon[155575]: Reconfiguring daemon crash.smithi167 on smithi167 2023-11-16T13:39:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:54.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi167.xxnzsz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-16T13:39:54.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-16T13:39:54.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:55.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:55 smithi167 ceph-mon[138945]: Reconfiguring mgr.smithi167.xxnzsz (monmap changed)... 2023-11-16T13:39:55.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:55 smithi167 ceph-mon[138945]: Reconfiguring daemon mgr.smithi167.xxnzsz on smithi167 2023-11-16T13:39:55.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:55 smithi167 ceph-mon[138945]: pgmap v16: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:39:55.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:55 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:55.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:55 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:55.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:55 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:39:55.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:55 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-11-16T13:39:55.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:55 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:55 smithi151 ceph-mon[155575]: Reconfiguring mgr.smithi167.xxnzsz (monmap changed)... 2023-11-16T13:39:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:55 smithi151 ceph-mon[155575]: Reconfiguring daemon mgr.smithi167.xxnzsz on smithi167 2023-11-16T13:39:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:55 smithi151 ceph-mon[155575]: pgmap v16: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:39:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:55 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:55 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:55 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-16T13:39:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:55 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-11-16T13:39:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:55 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:56.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:56 smithi167 ceph-mon[138945]: Reconfiguring mon.smithi167 (monmap changed)... 2023-11-16T13:39:56.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:56 smithi167 ceph-mon[138945]: Reconfiguring daemon mon.smithi167 on smithi167 2023-11-16T13:39:56.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:56 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:56.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:56 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:56.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:56 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-11-16T13:39:56.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:56 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:56.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:56 smithi151 ceph-mon[155575]: Reconfiguring mon.smithi167 (monmap changed)... 2023-11-16T13:39:56.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:56 smithi151 ceph-mon[155575]: Reconfiguring daemon mon.smithi167 on smithi167 2023-11-16T13:39:56.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:56 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:56.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:56 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:56.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:56 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-11-16T13:39:56.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:56 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:56.952 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:39:57.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:57 smithi167 ceph-mon[138945]: Reconfiguring osd.4 (monmap changed)... 2023-11-16T13:39:57.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:57 smithi167 ceph-mon[138945]: Reconfiguring daemon osd.4 on smithi167 2023-11-16T13:39:57.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:57 smithi167 ceph-mon[138945]: pgmap v17: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:39:57.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:57.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:57.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-11-16T13:39:57.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:57 smithi151 ceph-mon[155575]: Reconfiguring osd.4 (monmap changed)... 2023-11-16T13:39:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:57 smithi151 ceph-mon[155575]: Reconfiguring daemon osd.4 on smithi167 2023-11-16T13:39:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:57 smithi151 ceph-mon[155575]: pgmap v17: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:39:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-11-16T13:39:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:58.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:58 smithi167 ceph-mon[138945]: Reconfiguring osd.5 (monmap changed)... 2023-11-16T13:39:58.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:58 smithi167 ceph-mon[138945]: Reconfiguring daemon osd.5 on smithi167 2023-11-16T13:39:58.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:58 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:58.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:58 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:58.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:58 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-11-16T13:39:58.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:58 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:58.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:58 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:39:58.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:58 smithi151 ceph-mon[155575]: Reconfiguring osd.5 (monmap changed)... 2023-11-16T13:39:58.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:58 smithi151 ceph-mon[155575]: Reconfiguring daemon osd.5 on smithi167 2023-11-16T13:39:58.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:58 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:58.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:58 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:58.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:58 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-11-16T13:39:58.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:58 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:58.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:58 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:39:59.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:59 smithi167 ceph-mon[138945]: Reconfiguring osd.6 (monmap changed)... 2023-11-16T13:39:59.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:59 smithi167 ceph-mon[138945]: Reconfiguring daemon osd.6 on smithi167 2023-11-16T13:39:59.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:59 smithi167 ceph-mon[138945]: pgmap v18: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:59.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:59.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:59.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-11-16T13:39:59.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:39:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:39:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:59 smithi151 ceph-mon[155575]: Reconfiguring osd.6 (monmap changed)... 2023-11-16T13:39:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:59 smithi151 ceph-mon[155575]: Reconfiguring daemon osd.6 on smithi167 2023-11-16T13:39:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:59 smithi151 ceph-mon[155575]: pgmap v18: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:39:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:39:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-11-16T13:39:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:39:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:00.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:00 smithi151 ceph-mon[155575]: Reconfiguring osd.7 (monmap changed)... 2023-11-16T13:40:00.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:00 smithi151 ceph-mon[155575]: Reconfiguring daemon osd.7 on smithi167 2023-11-16T13:40:00.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:00 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:00.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:00 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:00.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:00 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi167.irznoa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-11-16T13:40:00.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:00 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:00.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:00 smithi151 ceph-mon[155575]: overall HEALTH_OK 2023-11-16T13:40:00.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:00 smithi167 ceph-mon[138945]: Reconfiguring osd.7 (monmap changed)... 2023-11-16T13:40:00.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:00 smithi167 ceph-mon[138945]: Reconfiguring daemon osd.7 on smithi167 2023-11-16T13:40:00.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:00 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:00.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:00 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:00.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:00 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi167.irznoa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-11-16T13:40:00.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:00 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:00.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:00 smithi167 ceph-mon[138945]: overall HEALTH_OK 2023-11-16T13:40:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:01 smithi151 ceph-mon[155575]: Reconfiguring mds.foofs.smithi167.irznoa (monmap changed)... 2023-11-16T13:40:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:01 smithi151 ceph-mon[155575]: Reconfiguring daemon mds.foofs.smithi167.irznoa on smithi167 2023-11-16T13:40:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:01 smithi151 ceph-mon[155575]: pgmap v19: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:01 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:01 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:01 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:01 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:01.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:01 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:01.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:01 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:01.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:01 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi151"}]: dispatch 2023-11-16T13:40:01.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:01 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi151"}]': finished 2023-11-16T13:40:01.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:01 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi167"}]: dispatch 2023-11-16T13:40:01.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:01 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi167"}]': finished 2023-11-16T13:40:01.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:01 smithi167 ceph-mon[138945]: Reconfiguring mds.foofs.smithi167.irznoa (monmap changed)... 2023-11-16T13:40:01.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:01 smithi167 ceph-mon[138945]: Reconfiguring daemon mds.foofs.smithi167.irznoa on smithi167 2023-11-16T13:40:01.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:01 smithi167 ceph-mon[138945]: pgmap v19: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:01.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:01 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:01.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:01 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:01.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:01 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:01.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:01 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:01.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:01 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:01.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:01 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:01.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:01 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi151"}]: dispatch 2023-11-16T13:40:01.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:01 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi151"}]': finished 2023-11-16T13:40:01.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:01 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi167"}]: dispatch 2023-11-16T13:40:01.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:01 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi167"}]': finished 2023-11-16T13:40:01.999 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:40:02.710 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:02 smithi151 ceph-mon[155575]: Upgrade: Setting container_image for all mon 2023-11-16T13:40:02.710 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:02.710 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi151", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:40:02.710 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:02.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:02 smithi167 ceph-mon[138945]: Upgrade: Setting container_image for all mon 2023-11-16T13:40:02.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:02.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi151", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:40:02.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:03 smithi151 ceph-mon[155575]: Upgrade: Updating crash.smithi151 (1/2) 2023-11-16T13:40:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:03 smithi151 ceph-mon[155575]: Deploying daemon crash.smithi151 on smithi151 2023-11-16T13:40:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:03 smithi151 ceph-mon[155575]: pgmap v20: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:03.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:03 smithi167 ceph-mon[138945]: Upgrade: Updating crash.smithi151 (1/2) 2023-11-16T13:40:03.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:03 smithi167 ceph-mon[138945]: Deploying daemon crash.smithi151 on smithi151 2023-11-16T13:40:03.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:03 smithi167 ceph-mon[138945]: pgmap v20: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:04.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:04 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:04.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:04 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:04.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:04 smithi167 ceph-mon[138945]: pgmap v21: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:04.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:04 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:04.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:04 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:04.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:04 smithi151 ceph-mon[155575]: pgmap v21: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:05.876 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:05 smithi167 ceph-mon[138945]: Upgrade: Updating crash.smithi167 (2/2) 2023-11-16T13:40:05.877 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:05 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:05.877 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:05 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi167", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:40:05.877 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:05 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:05.877 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:05 smithi167 ceph-mon[138945]: Deploying daemon crash.smithi167 on smithi167 2023-11-16T13:40:05.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:05 smithi151 ceph-mon[155575]: Upgrade: Updating crash.smithi167 (2/2) 2023-11-16T13:40:05.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:05 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:05.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:05 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi167", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-16T13:40:05.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:05 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:05.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:05 smithi151 ceph-mon[155575]: Deploying daemon crash.smithi167 on smithi167 2023-11-16T13:40:07.035 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:40:07.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:07 smithi167 ceph-mon[138945]: pgmap v22: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:07.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:07 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:07.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:07 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:07.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:07 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:07.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:07 smithi151 ceph-mon[155575]: pgmap v22: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:07.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:07 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:07.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:07 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:07.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:07 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:09.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:09 smithi151 ceph-mon[155575]: pgmap v23: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:09.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:09 smithi167 ceph-mon[138945]: pgmap v23: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:10.692 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:10 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:10.692 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:10 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:10.692 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:10 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:10.692 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:10 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:10 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:10 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:10 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:10 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:11.693 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:11 smithi151 ceph-mon[155575]: pgmap v24: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:11.694 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:11.694 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:11.694 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:11.694 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:11.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:11 smithi167 ceph-mon[138945]: pgmap v24: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:11.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:11.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:11.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:11.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:11.917 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:40:12.074 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:40:12.570 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:40:12.570 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (8m) 2s ago 9m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:40:12.571 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (9s) 2s ago 9m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:40:12.571 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (6s) 3s ago 8m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:40:12.571 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (8m) 2s ago 9m 36.2M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:40:12.571 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (5m) 2s ago 5m 16.0M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:40:12.571 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (5m) 3s ago 5m 42.4M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:40:12.571 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (81s) 2s ago 10m 490M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:40:12.571 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (61s) 3s ago 8m 438M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:40:12.571 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (51s) 2s ago 10m 74.5M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:40:12.571 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (34s) 3s ago 8m 66.2M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:40:12.571 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (3m) 2s ago 3m 81.5M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:40:12.571 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (3m) 3s ago 3m 56.2M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:40:12.571 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (9m) 2s ago 8m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:40:12.571 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (8m) 3s ago 8m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:40:12.572 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (8m) 2s ago 8m 1230M 1944M 16.2.5 6933c2a0b7dd 7598578fb701 2023-11-16T13:40:12.572 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (8m) 2s ago 8m 1081M 1944M 16.2.5 6933c2a0b7dd 989f26649fd3 2023-11-16T13:40:12.572 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (7m) 2s ago 7m 644M 1944M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:40:12.572 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (7m) 2s ago 7m 1161M 1944M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:40:12.572 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (7m) 3s ago 7m 1165M 2712M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:40:12.572 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (7m) 3s ago 7m 1364M 2712M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:40:12.572 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (6m) 3s ago 6m 1050M 2712M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:40:12.572 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (6m) 3s ago 6m 1816M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:40:12.572 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (84s) 2s ago 8m 56.9M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:40:12.950 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:40:12.950 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:40:12.950 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:40:12.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:40:12.950 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:40:12.950 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:40:12.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:40:12.950 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:40:12.951 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2023-11-16T13:40:12.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:40:12.951 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:40:12.951 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:40:12.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:40:12.951 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:40:12.951 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 10, 2023-11-16T13:40:12.951 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 4 2023-11-16T13:40:12.951 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:40:12.951 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:40:13.279 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:40:13.279 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:40:13.279 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:40:13.280 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:40:13.280 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:40:13.280 INFO:teuthology.orchestra.run.smithi151.stdout: "crash", 2023-11-16T13:40:13.280 INFO:teuthology.orchestra.run.smithi151.stdout: "mon", 2023-11-16T13:40:13.280 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:40:13.280 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:40:13.280 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "6/23 daemons upgraded", 2023-11-16T13:40:13.280 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading crash daemons", 2023-11-16T13:40:13.280 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:40:13.280 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:40:13.509 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:13 smithi151 ceph-mon[155575]: from='client.34150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:13.509 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:13 smithi151 ceph-mon[155575]: pgmap v25: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:13.509 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:13 smithi151 ceph-mon[155575]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:13.509 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:13.509 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:13.509 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:13 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/4085108614' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:13.509 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:40:13.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:13 smithi167 ceph-mon[138945]: from='client.34150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:13.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:13 smithi167 ceph-mon[138945]: pgmap v25: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:13.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:13 smithi167 ceph-mon[138945]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:13.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:13.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:13.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:13 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/4085108614' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:13.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:40:14.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:14.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='client.34166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:14.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:14.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:14.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:14.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:40:14.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:14.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:14.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:14.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:14.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:14.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:14.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:14.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi151"}]: dispatch 2023-11-16T13:40:14.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi151"}]': finished 2023-11-16T13:40:14.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi167"}]: dispatch 2023-11-16T13:40:14.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi167"}]': finished 2023-11-16T13:40:14.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-11-16T13:40:14.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:14.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='client.34166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:40:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi151"}]: dispatch 2023-11-16T13:40:14.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi151"}]': finished 2023-11-16T13:40:14.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi167"}]: dispatch 2023-11-16T13:40:14.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi167"}]': finished 2023-11-16T13:40:14.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-11-16T13:40:15.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:15 smithi151 ceph-mon[155575]: Upgrade: Setting container_image for all crash 2023-11-16T13:40:15.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:15 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-11-16T13:40:15.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:15 smithi151 ceph-mon[155575]: Upgrade: osd.0 is safe to restart 2023-11-16T13:40:15.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:15 smithi151 ceph-mon[155575]: pgmap v26: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:15.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:15.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-11-16T13:40:15.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:15.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:15 smithi167 ceph-mon[138945]: Upgrade: Setting container_image for all crash 2023-11-16T13:40:15.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:15 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-11-16T13:40:15.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:15 smithi167 ceph-mon[138945]: Upgrade: osd.0 is safe to restart 2023-11-16T13:40:15.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:15 smithi167 ceph-mon[138945]: pgmap v26: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:15.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:15.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-11-16T13:40:15.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:16.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:16 smithi151 ceph-mon[155575]: Upgrade: Updating osd.0 2023-11-16T13:40:16.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:16 smithi151 ceph-mon[155575]: Deploying daemon osd.0 on smithi151 2023-11-16T13:40:16.734 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:16 smithi151 systemd[1]: Stopping Ceph osd.0 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:40:16.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:16 smithi167 ceph-mon[138945]: Upgrade: Updating osd.0 2023-11-16T13:40:16.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:16 smithi167 ceph-mon[138945]: Deploying daemon osd.0 on smithi151 2023-11-16T13:40:17.115 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:40:17.198 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.0[117784]: debug 2023-11-16T13:40:16.769+0000 7faa39aad700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-11-16T13:40:17.198 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.0[117784]: debug 2023-11-16T13:40:16.769+0000 7faa39aad700 -1 osd.0 67 *** Got signal Terminated *** 2023-11-16T13:40:17.198 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:16 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.0[117784]: debug 2023-11-16T13:40:16.769+0000 7faa39aad700 -1 osd.0 67 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:40:17.198 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:17 smithi151 bash[162377]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.0 2023-11-16T13:40:17.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: pgmap v27: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:17.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.3 2023-11-16T13:40:17.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 failed (root=default,host=smithi151) (connection refused reported by osd.3) 2023-11-16T13:40:17.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.4 2023-11-16T13:40:17.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.1 2023-11-16T13:40:17.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.3 2023-11-16T13:40:17.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.2 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.3 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.1 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.2 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.1 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.6 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.4 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.6 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.4 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.7 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.5 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.7 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.5 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.3 2023-11-16T13:40:17.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.1 2023-11-16T13:40:17.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.4 2023-11-16T13:40:17.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:17 smithi151 ceph-mon[155575]: osd.0 reported immediately failed by osd.7 2023-11-16T13:40:17.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: pgmap v27: 129 pgs: 129 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:17.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.3 2023-11-16T13:40:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 failed (root=default,host=smithi151) (connection refused reported by osd.3) 2023-11-16T13:40:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.4 2023-11-16T13:40:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.1 2023-11-16T13:40:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.3 2023-11-16T13:40:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.2 2023-11-16T13:40:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.3 2023-11-16T13:40:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.1 2023-11-16T13:40:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.2 2023-11-16T13:40:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.1 2023-11-16T13:40:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.6 2023-11-16T13:40:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.4 2023-11-16T13:40:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.6 2023-11-16T13:40:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.4 2023-11-16T13:40:17.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.7 2023-11-16T13:40:17.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.5 2023-11-16T13:40:17.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.7 2023-11-16T13:40:17.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.5 2023-11-16T13:40:17.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.3 2023-11-16T13:40:17.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.1 2023-11-16T13:40:17.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.4 2023-11-16T13:40:17.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:17 smithi167 ceph-mon[138945]: osd.0 reported immediately failed by osd.7 2023-11-16T13:40:18.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:18 smithi151 ceph-mon[155575]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:40:18.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:18 smithi151 ceph-mon[155575]: osdmap e68: 8 total, 7 up, 8 in 2023-11-16T13:40:18.701 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:18 smithi151 systemd[1]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.0.service: Succeeded. 2023-11-16T13:40:18.701 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:18 smithi151 systemd[1]: Stopped Ceph osd.0 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:40:18.701 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:18 smithi151 systemd[1]: Starting Ceph osd.0 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:40:18.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:18 smithi167 ceph-mon[138945]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:40:18.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:18 smithi167 ceph-mon[138945]: osdmap e68: 8 total, 7 up, 8 in 2023-11-16T13:40:19.440 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:19 smithi151 bash[162672]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-11-16T13:40:19.440 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:19 smithi151 bash[162672]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2023-11-16T13:40:19.440 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:19 smithi151 bash[162672]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2023-11-16T13:40:19.440 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:19 smithi151 bash[162672]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-11-16T13:40:19.440 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:19 smithi151 bash[162672]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2023-11-16T13:40:19.440 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:19 smithi151 bash[162672]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-11-16T13:40:19.441 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:19 smithi151 bash[162672]: --> ceph-volume raw activate successful for osd ID: 0 2023-11-16T13:40:19.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:19 smithi151 ceph-mon[155575]: pgmap v29: 129 pgs: 16 stale+active+clean, 113 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:40:19.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:19 smithi151 ceph-mon[155575]: osdmap e69: 8 total, 7 up, 8 in 2023-11-16T13:40:19.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:19 smithi167 ceph-mon[138945]: pgmap v29: 129 pgs: 16 stale+active+clean, 113 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:40:19.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:19 smithi167 ceph-mon[138945]: osdmap e69: 8 total, 7 up, 8 in 2023-11-16T13:40:20.234 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:19 smithi151 ceph-osd[162961]: -- 172.21.15.151:0/3232525922 <== mon.0 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a386f9ab60 con 0x55a386f60c00 2023-11-16T13:40:20.234 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:19 smithi151 bash[162924]: 66e6cf45601a23559aee0e7eadbab9d5be957b63c8178cc43f931f868b69416d 2023-11-16T13:40:20.234 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:19 smithi151 systemd[1]: Started Ceph osd.0 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:40:20.734 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:20 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-0[162938]: 2023-11-16T13:40:20.388+0000 7f889ce77540 -1 Falling back to public interface 2023-11-16T13:40:21.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:21.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:21.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:21.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:20 smithi151 ceph-mon[155575]: pgmap v31: 129 pgs: 30 active+undersized+degraded, 19 active+undersized, 80 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 188/1389 objects degraded (13.535%) 2023-11-16T13:40:21.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:21.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:21.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:21.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:20 smithi167 ceph-mon[138945]: pgmap v31: 129 pgs: 30 active+undersized+degraded, 19 active+undersized, 80 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 188/1389 objects degraded (13.535%) 2023-11-16T13:40:22.150 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:40:27.399 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:21 smithi151 ceph-mon[155575]: Health check failed: Degraded data redundancy: 188/1389 objects degraded (13.535%), 30 pgs degraded (PG_DEGRADED) 2023-11-16T13:40:27.399 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:22 smithi151 ceph-mon[155575]: pgmap v32: 129 pgs: 30 active+undersized+degraded, 19 active+undersized, 80 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 188/1389 objects degraded (13.535%) 2023-11-16T13:40:27.400 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:24 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-0[162938]: 2023-11-16T13:40:24.721+0000 7f889ce77540 -1 osd.0 67 log_to_monitors true 2023-11-16T13:40:27.400 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:40:25 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-0[162938]: 2023-11-16T13:40:25.290+0000 7f888e457700 -1 osd.0 67 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-11-16T13:40:27.401 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:21 smithi167 ceph-mon[138945]: Health check failed: Degraded data redundancy: 188/1389 objects degraded (13.535%), 30 pgs degraded (PG_DEGRADED) 2023-11-16T13:40:27.401 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:22 smithi167 ceph-mon[138945]: pgmap v32: 129 pgs: 30 active+undersized+degraded, 19 active+undersized, 80 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 188/1389 objects degraded (13.535%) 2023-11-16T13:40:27.401 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:25 smithi167 ceph-mon[138945]: pgmap v33: 129 pgs: 30 active+undersized+degraded, 19 active+undersized, 80 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 188/1389 objects degraded (13.535%) 2023-11-16T13:40:27.401 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:27.401 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:27.402 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:25 smithi167 ceph-mon[138945]: from='osd.0 [v2:172.21.15.151:6802/3237051549,v1:172.21.15.151:6803/3237051549]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-11-16T13:40:27.433 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:27.433 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:27.433 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:26 smithi167 ceph-mon[138945]: from='osd.0 [v2:172.21.15.151:6802/3237051549,v1:172.21.15.151:6803/3237051549]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-11-16T13:40:27.433 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:26 smithi167 ceph-mon[138945]: osdmap e70: 8 total, 7 up, 8 in 2023-11-16T13:40:27.434 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:26 smithi167 ceph-mon[138945]: from='osd.0 [v2:172.21.15.151:6802/3237051549,v1:172.21.15.151:6803/3237051549]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:40:27.434 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:27 smithi167 ceph-mon[138945]: pgmap v35: 129 pgs: 30 active+undersized+degraded, 19 active+undersized, 80 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 188/1389 objects degraded (13.535%) 2023-11-16T13:40:27.434 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:27 smithi167 ceph-mon[138945]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:40:27.434 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:27 smithi167 ceph-mon[138945]: osd.0 [v2:172.21.15.151:6802/3237051549,v1:172.21.15.151:6803/3237051549] boot 2023-11-16T13:40:27.434 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:27 smithi167 ceph-mon[138945]: osdmap e71: 8 total, 8 up, 8 in 2023-11-16T13:40:27.435 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:40:27.435 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:25 smithi151 ceph-mon[155575]: pgmap v33: 129 pgs: 30 active+undersized+degraded, 19 active+undersized, 80 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 188/1389 objects degraded (13.535%) 2023-11-16T13:40:27.435 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:27.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:27.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:25 smithi151 ceph-mon[155575]: from='osd.0 [v2:172.21.15.151:6802/3237051549,v1:172.21.15.151:6803/3237051549]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-11-16T13:40:27.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:27.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:27.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:26 smithi151 ceph-mon[155575]: from='osd.0 [v2:172.21.15.151:6802/3237051549,v1:172.21.15.151:6803/3237051549]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-11-16T13:40:27.436 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:26 smithi151 ceph-mon[155575]: osdmap e70: 8 total, 7 up, 8 in 2023-11-16T13:40:27.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:26 smithi151 ceph-mon[155575]: from='osd.0 [v2:172.21.15.151:6802/3237051549,v1:172.21.15.151:6803/3237051549]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:40:27.449 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:40:27.449 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:27 smithi151 ceph-mon[155575]: pgmap v35: 129 pgs: 30 active+undersized+degraded, 19 active+undersized, 80 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 188/1389 objects degraded (13.535%) 2023-11-16T13:40:27.449 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:27 smithi151 ceph-mon[155575]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:40:27.450 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:27 smithi151 ceph-mon[155575]: osd.0 [v2:172.21.15.151:6802/3237051549,v1:172.21.15.151:6803/3237051549] boot 2023-11-16T13:40:27.450 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:27 smithi151 ceph-mon[155575]: osdmap e71: 8 total, 8 up, 8 in 2023-11-16T13:40:27.450 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-16T13:40:28.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: osdmap e72: 8 total, 8 up, 8 in 2023-11-16T13:40:28.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:28.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:28.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:28.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:40:28.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:28.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:28.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:28.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:28.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:28.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:28.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-11-16T13:40:28.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:28.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:40:28.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: osdmap e72: 8 total, 8 up, 8 in 2023-11-16T13:40:28.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:28.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:28.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:28.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:40:28.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:28.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:28.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:28.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:28.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:28.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:28.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-11-16T13:40:28.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:28.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:40:29.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:29 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-11-16T13:40:29.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:29 smithi167 ceph-mon[138945]: Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2023-11-16T13:40:29.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:29 smithi167 ceph-mon[138945]: pgmap v38: 129 pgs: 30 active+undersized+degraded, 19 active+undersized, 80 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 188/1389 objects degraded (13.535%) 2023-11-16T13:40:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:29 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-11-16T13:40:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:29 smithi151 ceph-mon[155575]: Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2023-11-16T13:40:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:29 smithi151 ceph-mon[155575]: pgmap v38: 129 pgs: 30 active+undersized+degraded, 19 active+undersized, 80 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 188/1389 objects degraded (13.535%) 2023-11-16T13:40:30.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:30 smithi151 ceph-mon[155575]: Health check update: Degraded data redundancy: 73/1389 objects degraded (5.256%), 9 pgs degraded (PG_DEGRADED) 2023-11-16T13:40:30.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:30 smithi167 ceph-mon[138945]: Health check update: Degraded data redundancy: 73/1389 objects degraded (5.256%), 9 pgs degraded (PG_DEGRADED) 2023-11-16T13:40:31.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:31 smithi151 ceph-mon[155575]: pgmap v39: 129 pgs: 9 active+undersized+degraded, 7 active+undersized, 113 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 73/1389 objects degraded (5.256%) 2023-11-16T13:40:31.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:31 smithi167 ceph-mon[138945]: pgmap v39: 129 pgs: 9 active+undersized+degraded, 7 active+undersized, 113 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 73/1389 objects degraded (5.256%) 2023-11-16T13:40:32.227 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:40:32.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:32 smithi151 ceph-mon[155575]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 73/1389 objects degraded (5.256%), 9 pgs degraded) 2023-11-16T13:40:32.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:32 smithi151 ceph-mon[155575]: Cluster is now healthy 2023-11-16T13:40:32.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:32 smithi167 ceph-mon[138945]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 73/1389 objects degraded (5.256%), 9 pgs degraded) 2023-11-16T13:40:32.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:32 smithi167 ceph-mon[138945]: Cluster is now healthy 2023-11-16T13:40:33.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:33 smithi151 ceph-mon[155575]: pgmap v40: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 153 B/s wr, 0 op/s 2023-11-16T13:40:33.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:33 smithi167 ceph-mon[138945]: pgmap v40: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 153 B/s wr, 0 op/s 2023-11-16T13:40:35.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:35 smithi151 ceph-mon[155575]: pgmap v41: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:35.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:35 smithi167 ceph-mon[138945]: pgmap v41: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:37.277 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:40:37.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:37 smithi151 ceph-mon[155575]: pgmap v42: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 211 B/s wr, 0 op/s 2023-11-16T13:40:37.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:37 smithi167 ceph-mon[138945]: pgmap v42: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 211 B/s wr, 0 op/s 2023-11-16T13:40:39.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:39 smithi167 ceph-mon[138945]: pgmap v43: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 191 B/s rd, 191 B/s wr, 0 op/s 2023-11-16T13:40:39.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:39.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:39 smithi151 ceph-mon[155575]: pgmap v43: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 191 B/s rd, 191 B/s wr, 0 op/s 2023-11-16T13:40:39.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:41.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:41 smithi167 ceph-mon[138945]: pgmap v44: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:41.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:41 smithi151 ceph-mon[155575]: pgmap v44: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:42.312 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:40:43.289 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:43 smithi151 ceph-mon[155575]: pgmap v45: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:43.289 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-11-16T13:40:43.289 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:43.289 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:40:43.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:43 smithi167 ceph-mon[138945]: pgmap v45: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:43.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-11-16T13:40:43.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:43.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:40:43.621 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:40:44.275 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:40:44.275 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (8m) 19s ago 9m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:40:44.275 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (41s) 19s ago 9m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:40:44.275 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (37s) 34s ago 9m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:40:44.275 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (8m) 19s ago 9m 36.2M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:40:44.275 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (6m) 19s ago 6m 16.0M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:40:44.276 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (6m) 34s ago 6m 42.4M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:40:44.276 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (113s) 19s ago 11m 493M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:40:44.276 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (92s) 34s ago 9m 438M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:40:44.276 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (83s) 19s ago 11m 77.9M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:40:44.276 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (66s) 34s ago 9m 66.2M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:40:44.276 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (4m) 19s ago 4m 81.5M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:40:44.276 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (4m) 34s ago 4m 56.2M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:40:44.276 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (9m) 19s ago 9m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:40:44.276 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (8m) 34s ago 8m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:40:44.276 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (24s) 19s ago 8m 12.0M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:40:44.276 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (8m) 19s ago 8m 1084M 1944M 16.2.5 6933c2a0b7dd 989f26649fd3 2023-11-16T13:40:44.277 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (8m) 19s ago 8m 646M 1944M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:40:44.277 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (8m) 19s ago 8m 1164M 1944M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:40:44.277 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (7m) 34s ago 7m 1165M 2712M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:40:44.277 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (7m) 34s ago 7m 1364M 2712M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:40:44.277 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (7m) 34s ago 7m 1050M 2712M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:40:44.277 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (7m) 34s ago 7m 1816M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:40:44.277 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (116s) 19s ago 9m 59.3M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:40:44.414 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:44 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-11-16T13:40:44.415 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:44 smithi151 ceph-mon[155575]: Upgrade: osd.1 is safe to restart 2023-11-16T13:40:44.415 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:44 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:44.415 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:44 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-11-16T13:40:44.415 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:44 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:44.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:44 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-11-16T13:40:44.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:44 smithi167 ceph-mon[138945]: Upgrade: osd.1 is safe to restart 2023-11-16T13:40:44.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:44 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:44.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:44 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-11-16T13:40:44.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:44 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:44.673 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:40:44.673 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:40:44.674 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:40:44.674 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:40:44.674 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:40:44.674 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:40:44.674 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:40:44.674 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:40:44.674 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 7, 2023-11-16T13:40:44.674 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 1 2023-11-16T13:40:44.674 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:40:44.674 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:40:44.674 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:40:44.675 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:40:44.675 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:40:44.675 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 9, 2023-11-16T13:40:44.675 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 5 2023-11-16T13:40:44.675 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:40:44.675 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:40:45.014 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:40:45.015 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:40:45.015 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:40:45.015 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:40:45.015 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:40:45.016 INFO:teuthology.orchestra.run.smithi151.stdout: "crash", 2023-11-16T13:40:45.016 INFO:teuthology.orchestra.run.smithi151.stdout: "mon", 2023-11-16T13:40:45.016 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:40:45.016 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:40:45.017 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "7/23 daemons upgraded", 2023-11-16T13:40:45.017 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading osd daemons", 2023-11-16T13:40:45.017 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:40:45.017 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:40:45.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:45 smithi151 ceph-mon[155575]: from='client.44121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:45.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:45 smithi151 ceph-mon[155575]: Upgrade: Updating osd.1 2023-11-16T13:40:45.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:45 smithi151 ceph-mon[155575]: Deploying daemon osd.1 on smithi151 2023-11-16T13:40:45.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:45 smithi151 ceph-mon[155575]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:45.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:45 smithi151 ceph-mon[155575]: pgmap v46: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:45.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:45 smithi151 ceph-mon[155575]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:45.416 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:45 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/698078823' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:45 smithi167 ceph-mon[138945]: from='client.44121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:45 smithi167 ceph-mon[138945]: Upgrade: Updating osd.1 2023-11-16T13:40:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:45 smithi167 ceph-mon[138945]: Deploying daemon osd.1 on smithi151 2023-11-16T13:40:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:45 smithi167 ceph-mon[138945]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:45.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:45 smithi167 ceph-mon[138945]: pgmap v46: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:40:45.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:45 smithi167 ceph-mon[138945]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:45.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:45 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/698078823' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:45.984 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:45 smithi151 systemd[1]: Stopping Ceph osd.1 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:40:45.984 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.1[121057]: debug 2023-11-16T13:40:45.781+0000 7ff21e6a6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-11-16T13:40:45.984 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.1[121057]: debug 2023-11-16T13:40:45.781+0000 7ff21e6a6700 -1 osd.1 72 *** Got signal Terminated *** 2023-11-16T13:40:45.984 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:45 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.1[121057]: debug 2023-11-16T13:40:45.781+0000 7ff21e6a6700 -1 osd.1 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:40:46.484 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:46 smithi151 bash[165566]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.1 2023-11-16T13:40:46.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: from='client.34190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:46.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.2 2023-11-16T13:40:46.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 failed (root=default,host=smithi151) (connection refused reported by osd.2) 2023-11-16T13:40:46.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.3 2023-11-16T13:40:46.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.6 2023-11-16T13:40:46.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.0 2023-11-16T13:40:46.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.2 2023-11-16T13:40:46.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.4 2023-11-16T13:40:46.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.7 2023-11-16T13:40:46.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.5 2023-11-16T13:40:46.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.3 2023-11-16T13:40:46.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.6 2023-11-16T13:40:46.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.0 2023-11-16T13:40:46.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.4 2023-11-16T13:40:46.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.7 2023-11-16T13:40:46.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:46 smithi151 ceph-mon[155575]: osd.1 reported immediately failed by osd.5 2023-11-16T13:40:46.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: from='client.34190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:40:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.2 2023-11-16T13:40:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 failed (root=default,host=smithi151) (connection refused reported by osd.2) 2023-11-16T13:40:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.3 2023-11-16T13:40:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.6 2023-11-16T13:40:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.0 2023-11-16T13:40:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.2 2023-11-16T13:40:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.4 2023-11-16T13:40:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.7 2023-11-16T13:40:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.5 2023-11-16T13:40:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.3 2023-11-16T13:40:46.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.6 2023-11-16T13:40:46.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.0 2023-11-16T13:40:46.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.4 2023-11-16T13:40:46.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.7 2023-11-16T13:40:46.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:46 smithi167 ceph-mon[138945]: osd.1 reported immediately failed by osd.5 2023-11-16T13:40:47.338 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:47 smithi151 ceph-mon[155575]: pgmap v47: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:47.339 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:47 smithi151 ceph-mon[155575]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:40:47.339 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:47 smithi151 ceph-mon[155575]: osdmap e73: 8 total, 7 up, 8 in 2023-11-16T13:40:47.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:47 smithi167 ceph-mon[138945]: pgmap v47: 129 pgs: 129 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:40:47.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:47 smithi167 ceph-mon[138945]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:40:47.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:47 smithi167 ceph-mon[138945]: osdmap e73: 8 total, 7 up, 8 in 2023-11-16T13:40:47.352 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:40:47.682 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:47 smithi151 systemd[1]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.1.service: Succeeded. 2023-11-16T13:40:47.682 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:47 smithi151 systemd[1]: Stopped Ceph osd.1 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:40:47.682 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:47 smithi151 systemd[1]: Starting Ceph osd.1 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:40:48.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:48 smithi151 ceph-mon[155575]: osdmap e74: 8 total, 7 up, 8 in 2023-11-16T13:40:48.484 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:48 smithi151 bash[165864]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-11-16T13:40:48.484 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:48 smithi151 bash[165864]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2023-11-16T13:40:48.484 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:48 smithi151 bash[165864]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2023-11-16T13:40:48.484 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:48 smithi151 bash[165864]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-11-16T13:40:48.485 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:48 smithi151 bash[165864]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2023-11-16T13:40:48.485 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:48 smithi151 bash[165864]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-11-16T13:40:48.485 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:48 smithi151 bash[165864]: --> ceph-volume raw activate successful for osd ID: 1 2023-11-16T13:40:48.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:48 smithi167 ceph-mon[138945]: osdmap e74: 8 total, 7 up, 8 in 2023-11-16T13:40:48.984 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:48 smithi151 ceph-osd[166176]: -- 172.21.15.151:0/3802339868 <== mon.0 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f1e072cb60 con 0x55f1e06f3000 2023-11-16T13:40:48.984 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:48 smithi151 bash[166139]: 49b15d0ef2d2da5de7e703b970852a5ecbc36b435cc8ad5bec7c169596a0dff1 2023-11-16T13:40:48.984 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:48 smithi151 systemd[1]: Started Ceph osd.1 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:40:49.484 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:49 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-1[166153]: 2023-11-16T13:40:49.333+0000 7f33812e6540 -1 Falling back to public interface 2023-11-16T13:40:49.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:49 smithi151 ceph-mon[155575]: pgmap v50: 129 pgs: 16 stale+active+clean, 113 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2023-11-16T13:40:49.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:49 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:49.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:49 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:49.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:49 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:49.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:49 smithi167 ceph-mon[138945]: pgmap v50: 129 pgs: 16 stale+active+clean, 113 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2023-11-16T13:40:49.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:49 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:49.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:49 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:49.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:49 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:50.353 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:50 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:50.354 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:50 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:50.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:50 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:50.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:50 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:51.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:51 smithi151 ceph-mon[155575]: pgmap v51: 129 pgs: 25 active+undersized+degraded, 1 stale+active+clean, 23 active+undersized, 80 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 148/1389 objects degraded (10.655%) 2023-11-16T13:40:51.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:51 smithi151 ceph-mon[155575]: Health check failed: Degraded data redundancy: 148/1389 objects degraded (10.655%), 25 pgs degraded (PG_DEGRADED) 2023-11-16T13:40:51.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:51 smithi167 ceph-mon[138945]: pgmap v51: 129 pgs: 25 active+undersized+degraded, 1 stale+active+clean, 23 active+undersized, 80 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 148/1389 objects degraded (10.655%) 2023-11-16T13:40:51.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:51 smithi167 ceph-mon[138945]: Health check failed: Degraded data redundancy: 148/1389 objects degraded (10.655%), 25 pgs degraded (PG_DEGRADED) 2023-11-16T13:40:52.397 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:40:53.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:53 smithi151 ceph-mon[155575]: pgmap v52: 129 pgs: 28 active+undersized+degraded, 25 active+undersized, 76 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 160/1389 objects degraded (11.519%) 2023-11-16T13:40:53.484 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:53 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-1[166153]: 2023-11-16T13:40:53.373+0000 7f33812e6540 -1 osd.1 72 log_to_monitors true 2023-11-16T13:40:53.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:53 smithi167 ceph-mon[138945]: pgmap v52: 129 pgs: 28 active+undersized+degraded, 25 active+undersized, 76 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 160/1389 objects degraded (11.519%) 2023-11-16T13:40:54.484 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:40:54 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-1[166153]: 2023-11-16T13:40:54.246+0000 7f33748ca700 -1 osd.1 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-11-16T13:40:54.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:54 smithi151 ceph-mon[155575]: from='osd.1 [v2:172.21.15.151:6810/1168122591,v1:172.21.15.151:6811/1168122591]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-11-16T13:40:54.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:54.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:54.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:54.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:54.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:54 smithi167 ceph-mon[138945]: from='osd.1 [v2:172.21.15.151:6810/1168122591,v1:172.21.15.151:6811/1168122591]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-11-16T13:40:54.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:54.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:54.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:54.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:55.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:55 smithi151 ceph-mon[155575]: pgmap v53: 129 pgs: 28 active+undersized+degraded, 25 active+undersized, 76 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 160/1389 objects degraded (11.519%) 2023-11-16T13:40:55.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:55 smithi151 ceph-mon[155575]: from='osd.1 [v2:172.21.15.151:6810/1168122591,v1:172.21.15.151:6811/1168122591]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-11-16T13:40:55.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:55 smithi151 ceph-mon[155575]: osdmap e75: 8 total, 7 up, 8 in 2023-11-16T13:40:55.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:55 smithi151 ceph-mon[155575]: from='osd.1 [v2:172.21.15.151:6810/1168122591,v1:172.21.15.151:6811/1168122591]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:40:55.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:55 smithi167 ceph-mon[138945]: pgmap v53: 129 pgs: 28 active+undersized+degraded, 25 active+undersized, 76 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 160/1389 objects degraded (11.519%) 2023-11-16T13:40:55.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:55 smithi167 ceph-mon[138945]: from='osd.1 [v2:172.21.15.151:6810/1168122591,v1:172.21.15.151:6811/1168122591]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-11-16T13:40:55.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:55 smithi167 ceph-mon[138945]: osdmap e75: 8 total, 7 up, 8 in 2023-11-16T13:40:55.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:55 smithi167 ceph-mon[138945]: from='osd.1 [v2:172.21.15.151:6810/1168122591,v1:172.21.15.151:6811/1168122591]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:40:56.277 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:56 smithi151 ceph-mon[155575]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:40:56.277 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:56 smithi151 ceph-mon[155575]: osd.1 [v2:172.21.15.151:6810/1168122591,v1:172.21.15.151:6811/1168122591] boot 2023-11-16T13:40:56.277 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:56 smithi151 ceph-mon[155575]: osdmap e76: 8 total, 8 up, 8 in 2023-11-16T13:40:56.277 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:56 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:40:56.277 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:56 smithi151 ceph-mon[155575]: Health check update: Degraded data redundancy: 160/1389 objects degraded (11.519%), 28 pgs degraded (PG_DEGRADED) 2023-11-16T13:40:56.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:56 smithi167 ceph-mon[138945]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:40:56.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:56 smithi167 ceph-mon[138945]: osd.1 [v2:172.21.15.151:6810/1168122591,v1:172.21.15.151:6811/1168122591] boot 2023-11-16T13:40:56.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:56 smithi167 ceph-mon[138945]: osdmap e76: 8 total, 8 up, 8 in 2023-11-16T13:40:56.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:56 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-11-16T13:40:56.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:56 smithi167 ceph-mon[138945]: Health check update: Degraded data redundancy: 160/1389 objects degraded (11.519%), 28 pgs degraded (PG_DEGRADED) 2023-11-16T13:40:57.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:57 smithi167 ceph-mon[138945]: pgmap v56: 129 pgs: 28 active+undersized+degraded, 25 active+undersized, 76 active+clean; 318 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 160/1389 objects degraded (11.519%) 2023-11-16T13:40:57.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:57 smithi167 ceph-mon[138945]: osdmap e77: 8 total, 8 up, 8 in 2023-11-16T13:40:57.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:57.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:57.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:57.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:40:57.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:57.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:57.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:57.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:57.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:57.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:57.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:57 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-11-16T13:40:57.446 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:40:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:57 smithi151 ceph-mon[155575]: pgmap v56: 129 pgs: 28 active+undersized+degraded, 25 active+undersized, 76 active+clean; 318 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 160/1389 objects degraded (11.519%) 2023-11-16T13:40:57.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:57 smithi151 ceph-mon[155575]: osdmap e77: 8 total, 8 up, 8 in 2023-11-16T13:40:57.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:57.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:57.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:40:57.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:40:57.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:57.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:57.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:40:57.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:57.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:57.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:40:57.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:57 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-11-16T13:40:58.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:58 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-11-16T13:40:58.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:58 smithi167 ceph-mon[138945]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-11-16T13:40:58.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:58 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:58.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:58 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:40:58.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:58 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-11-16T13:40:58.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:58 smithi151 ceph-mon[155575]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-11-16T13:40:58.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:58 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:40:58.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:58 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:40:59.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:40:59 smithi167 ceph-mon[138945]: pgmap v58: 129 pgs: 28 active+undersized+degraded, 25 active+undersized, 76 active+clean; 318 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 160/1389 objects degraded (11.519%) 2023-11-16T13:40:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:40:59 smithi151 ceph-mon[155575]: pgmap v58: 129 pgs: 28 active+undersized+degraded, 25 active+undersized, 76 active+clean; 318 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 160/1389 objects degraded (11.519%) 2023-11-16T13:41:00.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:00 smithi151 ceph-mon[155575]: mgrmap e40: smithi151.fcujra(active, since 91s), standbys: smithi167.xxnzsz 2023-11-16T13:41:00.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:00 smithi167 ceph-mon[138945]: mgrmap e40: smithi151.fcujra(active, since 91s), standbys: smithi167.xxnzsz 2023-11-16T13:41:01.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:01 smithi167 ceph-mon[138945]: pgmap v59: 129 pgs: 2 active+undersized+degraded, 2 active+undersized, 125 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 682 B/s wr, 0 op/s; 10/1389 objects degraded (0.720%) 2023-11-16T13:41:01.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:01 smithi167 ceph-mon[138945]: Health check update: Degraded data redundancy: 10/1389 objects degraded (0.720%), 2 pgs degraded (PG_DEGRADED) 2023-11-16T13:41:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:01 smithi151 ceph-mon[155575]: pgmap v59: 129 pgs: 2 active+undersized+degraded, 2 active+undersized, 125 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 682 B/s wr, 0 op/s; 10/1389 objects degraded (0.720%) 2023-11-16T13:41:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:01 smithi151 ceph-mon[155575]: Health check update: Degraded data redundancy: 10/1389 objects degraded (0.720%), 2 pgs degraded (PG_DEGRADED) 2023-11-16T13:41:02.479 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:41:02.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:02 smithi167 ceph-mon[138945]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/1389 objects degraded (0.720%), 2 pgs degraded) 2023-11-16T13:41:02.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:02 smithi167 ceph-mon[138945]: Cluster is now healthy 2023-11-16T13:41:02.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:02 smithi151 ceph-mon[155575]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/1389 objects degraded (0.720%), 2 pgs degraded) 2023-11-16T13:41:02.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:02 smithi151 ceph-mon[155575]: Cluster is now healthy 2023-11-16T13:41:03.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:03 smithi167 ceph-mon[138945]: pgmap v60: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 529 B/s wr, 0 op/s 2023-11-16T13:41:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:03 smithi151 ceph-mon[155575]: pgmap v60: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 529 B/s wr, 0 op/s 2023-11-16T13:41:05.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:05 smithi167 ceph-mon[138945]: pgmap v61: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 234 B/s rd, 585 B/s wr, 1 op/s 2023-11-16T13:41:05.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:05 smithi151 ceph-mon[155575]: pgmap v61: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 234 B/s rd, 585 B/s wr, 1 op/s 2023-11-16T13:41:07.522 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:41:07.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:07 smithi167 ceph-mon[138945]: pgmap v62: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 511 B/s wr, 0 op/s 2023-11-16T13:41:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:07 smithi151 ceph-mon[155575]: pgmap v62: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 511 B/s wr, 0 op/s 2023-11-16T13:41:09.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:09 smithi167 ceph-mon[138945]: pgmap v63: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 174 B/s rd, 436 B/s wr, 0 op/s 2023-11-16T13:41:09.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:09 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:09.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:09 smithi151 ceph-mon[155575]: pgmap v63: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 174 B/s rd, 436 B/s wr, 0 op/s 2023-11-16T13:41:09.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:09 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:11.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:11 smithi167 ceph-mon[138945]: pgmap v64: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2023-11-16T13:41:11.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:11 smithi151 ceph-mon[155575]: pgmap v64: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2023-11-16T13:41:12.437 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:12 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-11-16T13:41:12.557 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:41:12.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:12 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-11-16T13:41:13.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:13 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-11-16T13:41:13.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:13 smithi151 ceph-mon[155575]: Upgrade: osd.2 is safe to restart 2023-11-16T13:41:13.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:13 smithi151 ceph-mon[155575]: pgmap v65: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:41:13.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:13.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-11-16T13:41:13.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:41:13.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:13.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:41:13.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:13 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-11-16T13:41:13.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:13 smithi167 ceph-mon[138945]: Upgrade: osd.2 is safe to restart 2023-11-16T13:41:13.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:13 smithi167 ceph-mon[138945]: pgmap v65: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:41:13.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:13.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-11-16T13:41:13.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:41:13.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:13.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:41:14.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:14 smithi151 ceph-mon[155575]: Upgrade: Updating osd.2 2023-11-16T13:41:14.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:14 smithi151 ceph-mon[155575]: Deploying daemon osd.2 on smithi151 2023-11-16T13:41:14.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:14 smithi167 ceph-mon[138945]: Upgrade: Updating osd.2 2023-11-16T13:41:14.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:14 smithi167 ceph-mon[138945]: Deploying daemon osd.2 on smithi151 2023-11-16T13:41:14.984 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:14 smithi151 systemd[1]: Stopping Ceph osd.2 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:41:14.984 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.2[124902]: debug 2023-11-16T13:41:14.935+0000 7f152164b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-11-16T13:41:14.985 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.2[124902]: debug 2023-11-16T13:41:14.935+0000 7f152164b700 -1 osd.2 77 *** Got signal Terminated *** 2023-11-16T13:41:14.985 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:14 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.2[124902]: debug 2023-11-16T13:41:14.935+0000 7f152164b700 -1 osd.2 77 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:41:15.393 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:41:15.484 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:15 smithi151 bash[168766]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.2 2023-11-16T13:41:15.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: pgmap v66: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:41:15.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.4 2023-11-16T13:41:15.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 failed (root=default,host=smithi151) (connection refused reported by osd.4) 2023-11-16T13:41:15.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.7 2023-11-16T13:41:15.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.3 2023-11-16T13:41:15.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.3 2023-11-16T13:41:15.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.0 2023-11-16T13:41:15.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.1 2023-11-16T13:41:15.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.1 2023-11-16T13:41:15.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.0 2023-11-16T13:41:15.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.3 2023-11-16T13:41:15.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.6 2023-11-16T13:41:15.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.6 2023-11-16T13:41:15.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.4 2023-11-16T13:41:15.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.7 2023-11-16T13:41:15.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.5 2023-11-16T13:41:15.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.4 2023-11-16T13:41:15.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.7 2023-11-16T13:41:15.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:15 smithi151 ceph-mon[155575]: osd.2 reported immediately failed by osd.5 2023-11-16T13:41:15.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: pgmap v66: 129 pgs: 129 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.4 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 failed (root=default,host=smithi151) (connection refused reported by osd.4) 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.7 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.3 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.3 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.0 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.1 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.1 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.0 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.3 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.6 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.6 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.4 2023-11-16T13:41:15.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.7 2023-11-16T13:41:15.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.5 2023-11-16T13:41:15.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.4 2023-11-16T13:41:15.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.7 2023-11-16T13:41:15.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:15 smithi167 ceph-mon[138945]: osd.2 reported immediately failed by osd.5 2023-11-16T13:41:16.059 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:41:16.060 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (9m) 22s ago 10m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:41:16.060 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (72s) 22s ago 10m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:41:16.060 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (69s) 66s ago 9m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:41:16.060 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (9m) 22s ago 10m 36.2M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:41:16.060 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (6m) 22s ago 6m 16.1M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:41:16.060 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (6m) 66s ago 6m 42.4M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:41:16.060 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (2m) 22s ago 11m 497M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:41:16.060 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (2m) 66s ago 9m 438M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:41:16.060 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (115s) 22s ago 11m 82.2M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:41:16.060 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (98s) 66s ago 9m 66.2M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:41:16.060 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (4m) 22s ago 4m 83.4M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:41:16.061 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (4m) 66s ago 4m 56.2M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:41:16.061 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (10m) 22s ago 10m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:41:16.061 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (9m) 66s ago 9m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:41:16.061 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (56s) 22s ago 9m 369M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:41:16.061 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (27s) 22s ago 9m 12.2M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 49b15d0ef2d2 2023-11-16T13:41:16.061 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (8m) 22s ago 8m 651M 1944M 16.2.5 6933c2a0b7dd 44269265e025 2023-11-16T13:41:16.061 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (8m) 22s ago 8m 1170M 1944M 16.2.5 6933c2a0b7dd b013075b6407 2023-11-16T13:41:16.061 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (8m) 66s ago 8m 1165M 2712M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:41:16.061 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (8m) 66s ago 8m 1364M 2712M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:41:16.061 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (7m) 66s ago 7m 1050M 2712M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:41:16.062 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (7m) 66s ago 7m 1816M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:41:16.062 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (2m) 22s ago 9m 59.5M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:41:16.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:16 smithi151 ceph-mon[155575]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:41:16.417 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:16 smithi151 ceph-mon[155575]: osdmap e78: 8 total, 7 up, 8 in 2023-11-16T13:41:16.418 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:16 smithi151 ceph-mon[155575]: from='client.34198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:16.467 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:41:16.467 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:41:16.467 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:41:16.467 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:41:16.467 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:41:16.468 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:41:16.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:41:16.468 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:41:16.468 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 5, 2023-11-16T13:41:16.468 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:41:16.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:41:16.468 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:41:16.468 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:41:16.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:41:16.468 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:41:16.468 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 7, 2023-11-16T13:41:16.469 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 6 2023-11-16T13:41:16.469 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:41:16.469 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:41:16.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:16 smithi167 ceph-mon[138945]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:41:16.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:16 smithi167 ceph-mon[138945]: osdmap e78: 8 total, 7 up, 8 in 2023-11-16T13:41:16.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:16 smithi167 ceph-mon[138945]: from='client.34198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:16.734 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:16 smithi151 systemd[1]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.2.service: Succeeded. 2023-11-16T13:41:16.734 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:16 smithi151 systemd[1]: Stopped Ceph osd.2 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:41:16.734 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:16 smithi151 systemd[1]: Starting Ceph osd.2 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:41:16.817 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:41:16.817 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:41:16.817 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:41:16.817 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:41:16.817 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:41:16.817 INFO:teuthology.orchestra.run.smithi151.stdout: "crash", 2023-11-16T13:41:16.817 INFO:teuthology.orchestra.run.smithi151.stdout: "mon", 2023-11-16T13:41:16.817 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:41:16.817 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:41:16.817 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "8/23 daemons upgraded", 2023-11-16T13:41:16.818 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading osd daemons", 2023-11-16T13:41:16.818 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:41:16.818 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:41:17.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:17 smithi167 ceph-mon[138945]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:17.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:17 smithi167 ceph-mon[138945]: pgmap v68: 129 pgs: 5 peering, 9 stale+active+clean, 115 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:41:17.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:17 smithi167 ceph-mon[138945]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:17.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:17 smithi167 ceph-mon[138945]: osdmap e79: 8 total, 7 up, 8 in 2023-11-16T13:41:17.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:17 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/3340520504' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:17.467 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:17 smithi151 ceph-mon[155575]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:17.467 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:17 smithi151 ceph-mon[155575]: pgmap v68: 129 pgs: 5 peering, 9 stale+active+clean, 115 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:41:17.467 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:17 smithi151 ceph-mon[155575]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:17.467 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:17 smithi151 ceph-mon[155575]: osdmap e79: 8 total, 7 up, 8 in 2023-11-16T13:41:17.467 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:17 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/3340520504' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:17.467 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:17 smithi151 bash[169194]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-11-16T13:41:17.467 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:17 smithi151 bash[169194]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2023-11-16T13:41:17.467 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:17 smithi151 bash[169194]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2023-11-16T13:41:17.468 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:17 smithi151 bash[169194]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-11-16T13:41:17.468 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:17 smithi151 bash[169194]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2023-11-16T13:41:17.468 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:17 smithi151 bash[169194]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-11-16T13:41:17.468 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:17 smithi151 bash[169194]: --> ceph-volume raw activate successful for osd ID: 2 2023-11-16T13:41:17.593 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:41:18.202 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:17 smithi151 ceph-osd[169510]: -- 172.21.15.151:0/3708718667 <== mon.0 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c1301f0b60 con 0x55c1301b6c00 2023-11-16T13:41:18.202 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:17 smithi151 bash[169473]: 89fcacc46719062330a61e4660a33f1ed0bbaf762f7055cdb0dc82ea2d086c87 2023-11-16T13:41:18.202 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:17 smithi151 systemd[1]: Started Ceph osd.2 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:41:18.464 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:18 smithi151 ceph-mon[155575]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:18.464 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:18.464 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:18.464 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:41:18.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:18 smithi167 ceph-mon[138945]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:18.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:18.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:18.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:41:18.734 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:18 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-2[169487]: 2023-11-16T13:41:18.461+0000 7f557df99540 -1 Falling back to public interface 2023-11-16T13:41:19.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:19 smithi151 ceph-mon[155575]: pgmap v70: 129 pgs: 5 peering, 9 stale+active+clean, 115 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2023-11-16T13:41:19.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:19 smithi167 ceph-mon[138945]: pgmap v70: 129 pgs: 5 peering, 9 stale+active+clean, 115 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2023-11-16T13:41:20.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:20 smithi167 ceph-mon[138945]: Health check failed: Degraded data redundancy: 117/1389 objects degraded (8.423%), 20 pgs degraded (PG_DEGRADED) 2023-11-16T13:41:20.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:20 smithi151 ceph-mon[155575]: Health check failed: Degraded data redundancy: 117/1389 objects degraded (8.423%), 20 pgs degraded (PG_DEGRADED) 2023-11-16T13:41:21.461 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:21 smithi151 ceph-mon[155575]: pgmap v71: 129 pgs: 20 active+undersized+degraded, 5 peering, 10 active+undersized, 94 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 117/1389 objects degraded (8.423%) 2023-11-16T13:41:22.196 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:21 smithi167 ceph-mon[138945]: pgmap v71: 129 pgs: 20 active+undersized+degraded, 5 peering, 10 active+undersized, 94 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 117/1389 objects degraded (8.423%) 2023-11-16T13:41:22.626 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:41:22.653 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-2[169487]: 2023-11-16T13:41:22.273+0000 7f557df99540 -1 osd.2 77 log_to_monitors true 2023-11-16T13:41:22.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:22 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:22.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:22 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:22.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:22 smithi151 ceph-mon[155575]: pgmap v72: 129 pgs: 21 active+undersized+degraded, 14 active+undersized, 94 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 131/1389 objects degraded (9.431%) 2023-11-16T13:41:22.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:22 smithi151 ceph-mon[155575]: from='osd.2 [v2:172.21.15.151:6818/1569369535,v1:172.21.15.151:6819/1569369535]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-11-16T13:41:22.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:22 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:22.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:22 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:22.985 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:41:22 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-2[169487]: 2023-11-16T13:41:22.671+0000 7f557157d700 -1 osd.2 77 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-11-16T13:41:23.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:22 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:23.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:22 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:23.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:22 smithi167 ceph-mon[138945]: pgmap v72: 129 pgs: 21 active+undersized+degraded, 14 active+undersized, 94 active+clean; 318 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 131/1389 objects degraded (9.431%) 2023-11-16T13:41:23.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:22 smithi167 ceph-mon[138945]: from='osd.2 [v2:172.21.15.151:6818/1569369535,v1:172.21.15.151:6819/1569369535]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-11-16T13:41:23.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:22 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:23.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:22 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:23.670 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:23 smithi151 ceph-mon[155575]: from='osd.2 [v2:172.21.15.151:6818/1569369535,v1:172.21.15.151:6819/1569369535]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-11-16T13:41:23.670 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:23 smithi151 ceph-mon[155575]: osdmap e80: 8 total, 7 up, 8 in 2023-11-16T13:41:23.671 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:23 smithi151 ceph-mon[155575]: from='osd.2 [v2:172.21.15.151:6818/1569369535,v1:172.21.15.151:6819/1569369535]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:41:24.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:23 smithi167 ceph-mon[138945]: from='osd.2 [v2:172.21.15.151:6818/1569369535,v1:172.21.15.151:6819/1569369535]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-11-16T13:41:24.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:23 smithi167 ceph-mon[138945]: osdmap e80: 8 total, 7 up, 8 in 2023-11-16T13:41:24.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:23 smithi167 ceph-mon[138945]: from='osd.2 [v2:172.21.15.151:6818/1569369535,v1:172.21.15.151:6819/1569369535]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:41:24.868 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:24 smithi151 ceph-mon[155575]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:41:24.868 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:24 smithi151 ceph-mon[155575]: osd.2 [v2:172.21.15.151:6818/1569369535,v1:172.21.15.151:6819/1569369535] boot 2023-11-16T13:41:24.868 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:24 smithi151 ceph-mon[155575]: osdmap e81: 8 total, 8 up, 8 in 2023-11-16T13:41:24.869 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:24 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:41:24.869 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:24 smithi151 ceph-mon[155575]: pgmap v75: 129 pgs: 17 peering, 9 active+undersized+degraded, 9 active+undersized, 94 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 131 B/s wr, 0 op/s; 66/1389 objects degraded (4.752%) 2023-11-16T13:41:24.869 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:24 smithi151 ceph-mon[155575]: osdmap e82: 8 total, 8 up, 8 in 2023-11-16T13:41:25.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:24 smithi167 ceph-mon[138945]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:41:25.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:24 smithi167 ceph-mon[138945]: osd.2 [v2:172.21.15.151:6818/1569369535,v1:172.21.15.151:6819/1569369535] boot 2023-11-16T13:41:25.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:24 smithi167 ceph-mon[138945]: osdmap e81: 8 total, 8 up, 8 in 2023-11-16T13:41:25.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:24 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-11-16T13:41:25.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:24 smithi167 ceph-mon[138945]: pgmap v75: 129 pgs: 17 peering, 9 active+undersized+degraded, 9 active+undersized, 94 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 131 B/s wr, 0 op/s; 66/1389 objects degraded (4.752%) 2023-11-16T13:41:25.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:24 smithi167 ceph-mon[138945]: osdmap e82: 8 total, 8 up, 8 in 2023-11-16T13:41:26.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:26.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:26.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:41:26.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:41:26.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:26.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:26.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:41:26.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:26.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:26.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:26.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-11-16T13:41:26.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:25 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-11-16T13:41:26.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:25 smithi151 ceph-mon[155575]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2023-11-16T13:41:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:41:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:41:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:41:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:26.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:26.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-11-16T13:41:26.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:25 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-11-16T13:41:26.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:25 smithi167 ceph-mon[138945]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2023-11-16T13:41:27.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:26 smithi151 ceph-mon[155575]: pgmap v77: 129 pgs: 21 peering, 5 active+undersized+degraded, 6 active+undersized, 97 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 24/1389 objects degraded (1.728%) 2023-11-16T13:41:27.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:26 smithi151 ceph-mon[155575]: Health check update: Degraded data redundancy: 66/1389 objects degraded (4.752%), 9 pgs degraded (PG_DEGRADED) 2023-11-16T13:41:27.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:26 smithi167 ceph-mon[138945]: pgmap v77: 129 pgs: 21 peering, 5 active+undersized+degraded, 6 active+undersized, 97 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 24/1389 objects degraded (1.728%) 2023-11-16T13:41:27.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:26 smithi167 ceph-mon[138945]: Health check update: Degraded data redundancy: 66/1389 objects degraded (4.752%), 9 pgs degraded (PG_DEGRADED) 2023-11-16T13:41:27.669 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:41:29.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:29 smithi167 ceph-mon[138945]: pgmap v78: 129 pgs: 21 peering, 108 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:29 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:29 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:41:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:29 smithi167 ceph-mon[138945]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/1389 objects degraded (1.728%), 5 pgs degraded) 2023-11-16T13:41:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:29 smithi167 ceph-mon[138945]: Cluster is now healthy 2023-11-16T13:41:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:29 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:29 smithi151 ceph-mon[155575]: pgmap v78: 129 pgs: 21 peering, 108 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:29 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:29 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:41:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:29 smithi151 ceph-mon[155575]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/1389 objects degraded (1.728%), 5 pgs degraded) 2023-11-16T13:41:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:29 smithi151 ceph-mon[155575]: Cluster is now healthy 2023-11-16T13:41:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:29 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:31.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:31 smithi167 ceph-mon[138945]: pgmap v79: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 139 B/s rd, 279 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:31.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:31 smithi151 ceph-mon[155575]: pgmap v79: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 139 B/s rd, 279 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:32.708 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:41:33.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:33 smithi151 ceph-mon[155575]: pgmap v80: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 123 B/s rd, 246 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:33.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:33 smithi167 ceph-mon[138945]: pgmap v80: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 123 B/s rd, 246 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:35.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:35 smithi151 ceph-mon[155575]: pgmap v81: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:35.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:35 smithi167 ceph-mon[138945]: pgmap v81: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:37.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:37 smithi167 ceph-mon[138945]: pgmap v82: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 180 B/s rd, 180 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:37.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:37 smithi151 ceph-mon[155575]: pgmap v82: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 180 B/s rd, 180 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:37.748 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:41:39.350 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:39 smithi167 ceph-mon[138945]: pgmap v83: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:39.350 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:39.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:39 smithi151 ceph-mon[155575]: pgmap v83: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:39.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:41.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:41 smithi167 ceph-mon[138945]: pgmap v84: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2023-11-16T13:41:41.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-11-16T13:41:41.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:41 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-11-16T13:41:41.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:41 smithi167 ceph-mon[138945]: Upgrade: osd.3 is safe to restart 2023-11-16T13:41:41.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:41 smithi151 ceph-mon[155575]: pgmap v84: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2023-11-16T13:41:41.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-11-16T13:41:41.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:41 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-11-16T13:41:41.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:41 smithi151 ceph-mon[155575]: Upgrade: osd.3 is safe to restart 2023-11-16T13:41:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:42 smithi151 ceph-mon[155575]: Upgrade: Updating osd.3 2023-11-16T13:41:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:42.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-11-16T13:41:42.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:41:42.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:42 smithi151 ceph-mon[155575]: Deploying daemon osd.3 on smithi151 2023-11-16T13:41:42.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:42 smithi167 ceph-mon[138945]: Upgrade: Updating osd.3 2023-11-16T13:41:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-11-16T13:41:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:41:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:42 smithi167 ceph-mon[138945]: Deploying daemon osd.3 on smithi151 2023-11-16T13:41:42.794 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:41:43.371 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:43 smithi151 ceph-mon[155575]: pgmap v85: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2023-11-16T13:41:43.371 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:43.372 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:41:43.372 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:43 smithi151 systemd[1]: Stopping Ceph osd.3 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:41:43.372 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.3[128565]: debug 2023-11-16T13:41:43.205+0000 7f12ca784700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-11-16T13:41:43.372 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.3[128565]: debug 2023-11-16T13:41:43.205+0000 7f12ca784700 -1 osd.3 82 *** Got signal Terminated *** 2023-11-16T13:41:43.372 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.3[128565]: debug 2023-11-16T13:41:43.205+0000 7f12ca784700 -1 osd.3 82 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:41:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:43 smithi167 ceph-mon[138945]: pgmap v85: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2023-11-16T13:41:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:41:43.734 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:43 smithi151 bash[171988]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.3 2023-11-16T13:41:44.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.0 2023-11-16T13:41:44.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 failed (root=default,host=smithi151) (connection refused reported by osd.0) 2023-11-16T13:41:44.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.0 2023-11-16T13:41:44.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.1 2023-11-16T13:41:44.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.2 2023-11-16T13:41:44.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.4 2023-11-16T13:41:44.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.7 2023-11-16T13:41:44.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.5 2023-11-16T13:41:44.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.6 2023-11-16T13:41:44.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.1 2023-11-16T13:41:44.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.2 2023-11-16T13:41:44.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.4 2023-11-16T13:41:44.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.7 2023-11-16T13:41:44.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.5 2023-11-16T13:41:44.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.6 2023-11-16T13:41:44.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.2 2023-11-16T13:41:44.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.2 2023-11-16T13:41:44.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.4 2023-11-16T13:41:44.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.7 2023-11-16T13:41:44.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.5 2023-11-16T13:41:44.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.6 2023-11-16T13:41:44.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.0 2023-11-16T13:41:44.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.1 2023-11-16T13:41:44.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.4 2023-11-16T13:41:44.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.7 2023-11-16T13:41:44.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.5 2023-11-16T13:41:44.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.6 2023-11-16T13:41:44.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.0 2023-11-16T13:41:44.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.1 2023-11-16T13:41:44.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.6 2023-11-16T13:41:44.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.5 2023-11-16T13:41:44.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.7 2023-11-16T13:41:44.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.6 2023-11-16T13:41:44.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.5 2023-11-16T13:41:44.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:44 smithi151 ceph-mon[155575]: osd.3 reported immediately failed by osd.7 2023-11-16T13:41:44.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.0 2023-11-16T13:41:44.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 failed (root=default,host=smithi151) (connection refused reported by osd.0) 2023-11-16T13:41:44.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.0 2023-11-16T13:41:44.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.1 2023-11-16T13:41:44.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.2 2023-11-16T13:41:44.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.4 2023-11-16T13:41:44.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.7 2023-11-16T13:41:44.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.5 2023-11-16T13:41:44.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.6 2023-11-16T13:41:44.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.1 2023-11-16T13:41:44.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.2 2023-11-16T13:41:44.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.4 2023-11-16T13:41:44.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.7 2023-11-16T13:41:44.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.5 2023-11-16T13:41:44.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.6 2023-11-16T13:41:44.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.2 2023-11-16T13:41:44.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.2 2023-11-16T13:41:44.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.4 2023-11-16T13:41:44.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.7 2023-11-16T13:41:44.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.5 2023-11-16T13:41:44.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.6 2023-11-16T13:41:44.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.0 2023-11-16T13:41:44.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.1 2023-11-16T13:41:44.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.4 2023-11-16T13:41:44.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.7 2023-11-16T13:41:44.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.5 2023-11-16T13:41:44.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.6 2023-11-16T13:41:44.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.0 2023-11-16T13:41:44.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.1 2023-11-16T13:41:44.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.6 2023-11-16T13:41:44.601 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.5 2023-11-16T13:41:44.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.7 2023-11-16T13:41:44.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.6 2023-11-16T13:41:44.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.5 2023-11-16T13:41:44.602 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:44 smithi167 ceph-mon[138945]: osd.3 reported immediately failed by osd.7 2023-11-16T13:41:44.956 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:44 smithi151 systemd[1]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.3.service: Succeeded. 2023-11-16T13:41:44.956 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:44 smithi151 systemd[1]: Stopped Ceph osd.3 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:41:45.227 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:44 smithi151 systemd[1]: Starting Ceph osd.3 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:41:45.227 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:45 smithi151 ceph-mon[155575]: pgmap v86: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:41:45.228 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:45 smithi151 ceph-mon[155575]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:41:45.228 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:45 smithi151 ceph-mon[155575]: osdmap e83: 8 total, 7 up, 8 in 2023-11-16T13:41:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:45 smithi167 ceph-mon[138945]: pgmap v86: 129 pgs: 129 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:41:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:45 smithi167 ceph-mon[138945]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:41:45.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:45 smithi167 ceph-mon[138945]: osdmap e83: 8 total, 7 up, 8 in 2023-11-16T13:41:45.984 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:45 smithi151 bash[172266]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-11-16T13:41:45.984 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:45 smithi151 bash[172266]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2023-11-16T13:41:45.984 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:45 smithi151 bash[172266]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2023-11-16T13:41:45.984 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:45 smithi151 bash[172266]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-11-16T13:41:45.984 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:45 smithi151 bash[172266]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2023-11-16T13:41:45.984 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:45 smithi151 bash[172266]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-11-16T13:41:45.984 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:45 smithi151 bash[172266]: --> ceph-volume raw activate successful for osd ID: 3 2023-11-16T13:41:46.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:46 smithi151 ceph-mon[155575]: osdmap e84: 8 total, 7 up, 8 in 2023-11-16T13:41:46.484 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:46 smithi151 ceph-osd[172554]: -- 172.21.15.151:0/324260724 <== mon.0 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556103e3eb60 con 0x556103e04c00 2023-11-16T13:41:46.484 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:46 smithi151 bash[172517]: aa3cf87e766a169742a12442abb88ee8d0ceb4b960d9e3ce92f44dab15eca743 2023-11-16T13:41:46.484 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:46 smithi151 systemd[1]: Started Ceph osd.3 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:41:46.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:46 smithi167 ceph-mon[138945]: osdmap e84: 8 total, 7 up, 8 in 2023-11-16T13:41:47.168 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:41:47.208 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:46 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-3[172531]: 2023-11-16T13:41:46.870+0000 7f0358ff8540 -1 Falling back to public interface 2023-11-16T13:41:47.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:47 smithi167 ceph-mon[138945]: pgmap v89: 129 pgs: 7 active+undersized+degraded, 10 peering, 11 stale+active+clean, 3 active+undersized, 98 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 50/1389 objects degraded (3.600%) 2023-11-16T13:41:47.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:47 smithi167 ceph-mon[138945]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-11-16T13:41:47.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:47 smithi167 ceph-mon[138945]: Health check failed: Degraded data redundancy: 50/1389 objects degraded (3.600%), 7 pgs degraded (PG_DEGRADED) 2023-11-16T13:41:47.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:47.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:47.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:41:47.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:47.350 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:47.481 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:47 smithi151 ceph-mon[155575]: pgmap v89: 129 pgs: 7 active+undersized+degraded, 10 peering, 11 stale+active+clean, 3 active+undersized, 98 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 50/1389 objects degraded (3.600%) 2023-11-16T13:41:47.481 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:47 smithi151 ceph-mon[155575]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-11-16T13:41:47.481 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:47 smithi151 ceph-mon[155575]: Health check failed: Degraded data redundancy: 50/1389 objects degraded (3.600%), 7 pgs degraded (PG_DEGRADED) 2023-11-16T13:41:47.481 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:47.481 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:47.482 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:41:47.482 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:47.482 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:47.836 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:41:47.869 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:41:47.869 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (9m) 26s ago 11m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:41:47.869 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (104s) 26s ago 11m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:41:47.869 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (101s) 98s ago 10m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:41:47.869 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (9m) 26s ago 10m 36.4M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:41:47.869 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (7m) 26s ago 7m 16.2M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:41:47.869 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (7m) 98s ago 7m 42.4M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:41:47.870 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (2m) 26s ago 12m 498M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:41:47.870 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (2m) 98s ago 10m 438M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:41:47.870 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (2m) 26s ago 12m 87.5M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:41:47.870 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (2m) 98s ago 10m 66.2M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:41:47.870 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (5m) 26s ago 5m 83.5M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:41:47.870 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (5m) 98s ago 5m 56.2M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:41:47.870 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (10m) 26s ago 10m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:41:47.870 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (10m) 98s ago 10m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:41:47.870 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (88s) 26s ago 9m 376M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:41:47.870 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (59s) 26s ago 9m 284M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 49b15d0ef2d2 2023-11-16T13:41:47.870 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (30s) 26s ago 9m 14.3M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 89fcacc46719 2023-11-16T13:41:47.870 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 starting - - - 1944M 2023-11-16T13:41:47.871 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (8m) 98s ago 8m 1165M 2712M 16.2.5 6933c2a0b7dd a0e73aaa308c 2023-11-16T13:41:47.871 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (8m) 98s ago 8m 1364M 2712M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:41:47.871 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (8m) 98s ago 8m 1050M 2712M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:41:47.871 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (8m) 98s ago 8m 1816M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:41:47.871 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (2m) 26s ago 10m 59.7M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:41:48.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:48 smithi151 ceph-mon[155575]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:48.274 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:41:48.275 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:41:48.275 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:41:48.276 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:41:48.276 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:41:48.277 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:41:48.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:41:48.277 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:41:48.277 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 4, 2023-11-16T13:41:48.277 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 3 2023-11-16T13:41:48.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:41:48.277 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:41:48.277 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:41:48.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:41:48.278 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:41:48.278 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 6, 2023-11-16T13:41:48.278 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 7 2023-11-16T13:41:48.278 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:41:48.278 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:41:48.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:48 smithi167 ceph-mon[138945]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:48.603 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:41:48.603 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:41:48.603 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:41:48.603 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:41:48.603 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:41:48.603 INFO:teuthology.orchestra.run.smithi151.stdout: "crash", 2023-11-16T13:41:48.604 INFO:teuthology.orchestra.run.smithi151.stdout: "mon", 2023-11-16T13:41:48.604 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:41:48.604 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:41:48.604 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "9/23 daemons upgraded", 2023-11-16T13:41:48.604 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading osd daemons", 2023-11-16T13:41:48.604 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:41:48.604 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:41:49.215 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:49 smithi151 ceph-mon[155575]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:49.217 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:49 smithi151 ceph-mon[155575]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:49.217 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:49 smithi151 ceph-mon[155575]: pgmap v90: 129 pgs: 11 active+undersized+degraded, 10 peering, 9 stale+active+clean, 6 active+undersized, 93 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 61/1389 objects degraded (4.392%) 2023-11-16T13:41:49.217 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:49 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/1696679306' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:49.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:49 smithi167 ceph-mon[138945]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:49.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:49 smithi167 ceph-mon[138945]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:49.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:49 smithi167 ceph-mon[138945]: pgmap v90: 129 pgs: 11 active+undersized+degraded, 10 peering, 9 stale+active+clean, 6 active+undersized, 93 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 61/1389 objects degraded (4.392%) 2023-11-16T13:41:49.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:49 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/1696679306' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:50.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:50 smithi151 ceph-mon[155575]: from='client.34240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:50.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:50 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:50.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:50 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:50.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:50 smithi167 ceph-mon[138945]: from='client.34240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:41:50.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:50 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:50.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:50 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:51.484 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:51 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-3[172531]: 2023-11-16T13:41:51.135+0000 7f0358ff8540 -1 osd.3 82 log_to_monitors true 2023-11-16T13:41:51.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:51 smithi151 ceph-mon[155575]: pgmap v91: 129 pgs: 30 active+undersized+degraded, 20 active+undersized, 79 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 161/1389 objects degraded (11.591%) 2023-11-16T13:41:51.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:51 smithi151 ceph-mon[155575]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2023-11-16T13:41:51.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:51 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:51.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:51 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:51.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:51 smithi151 ceph-mon[155575]: from='osd.3 [v2:172.21.15.151:6826/2437912170,v1:172.21.15.151:6827/2437912170]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-11-16T13:41:51.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:51 smithi167 ceph-mon[138945]: pgmap v91: 129 pgs: 30 active+undersized+degraded, 20 active+undersized, 79 active+clean; 318 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 161/1389 objects degraded (11.591%) 2023-11-16T13:41:51.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:51 smithi167 ceph-mon[138945]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2023-11-16T13:41:51.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:51 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:51.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:51 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:51.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:51 smithi167 ceph-mon[138945]: from='osd.3 [v2:172.21.15.151:6826/2437912170,v1:172.21.15.151:6827/2437912170]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-11-16T13:41:52.234 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:41:51 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-3[172531]: 2023-11-16T13:41:51.916+0000 7f034b5da700 -1 osd.3 82 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-11-16T13:41:52.874 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:41:53.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:52 smithi151 ceph-mon[155575]: from='osd.3 [v2:172.21.15.151:6826/2437912170,v1:172.21.15.151:6827/2437912170]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-11-16T13:41:53.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:52 smithi151 ceph-mon[155575]: osdmap e85: 8 total, 7 up, 8 in 2023-11-16T13:41:53.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:52 smithi151 ceph-mon[155575]: from='osd.3 [v2:172.21.15.151:6826/2437912170,v1:172.21.15.151:6827/2437912170]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:41:53.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:52 smithi151 ceph-mon[155575]: pgmap v93: 129 pgs: 30 active+undersized+degraded, 20 active+undersized, 79 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 131 B/s wr, 0 op/s; 161/1389 objects degraded (11.591%) 2023-11-16T13:41:53.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:52 smithi167 ceph-mon[138945]: from='osd.3 [v2:172.21.15.151:6826/2437912170,v1:172.21.15.151:6827/2437912170]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-11-16T13:41:53.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:52 smithi167 ceph-mon[138945]: osdmap e85: 8 total, 7 up, 8 in 2023-11-16T13:41:53.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:52 smithi167 ceph-mon[138945]: from='osd.3 [v2:172.21.15.151:6826/2437912170,v1:172.21.15.151:6827/2437912170]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-11-16T13:41:53.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:52 smithi167 ceph-mon[138945]: pgmap v93: 129 pgs: 30 active+undersized+degraded, 20 active+undersized, 79 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 131 B/s wr, 0 op/s; 161/1389 objects degraded (11.591%) 2023-11-16T13:41:54.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:41:54.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: osd.3 [v2:172.21.15.151:6826/2437912170,v1:172.21.15.151:6827/2437912170] boot 2023-11-16T13:41:54.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: osdmap e86: 8 total, 8 up, 8 in 2023-11-16T13:41:54.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:41:54.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:54.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:54.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:41:54.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:41:54.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:54.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:54.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:41:54.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:54.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:54.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:54.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-11-16T13:41:54.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-11-16T13:41:54.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:53 smithi151 ceph-mon[155575]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2023-11-16T13:41:54.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:41:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: osd.3 [v2:172.21.15.151:6826/2437912170,v1:172.21.15.151:6827/2437912170] boot 2023-11-16T13:41:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: osdmap e86: 8 total, 8 up, 8 in 2023-11-16T13:41:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-11-16T13:41:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:41:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:41:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:54.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:41:54.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:54.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:54.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:41:54.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-11-16T13:41:54.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-11-16T13:41:54.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:53 smithi167 ceph-mon[138945]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2023-11-16T13:41:55.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:54 smithi151 ceph-mon[155575]: osdmap e87: 8 total, 8 up, 8 in 2023-11-16T13:41:55.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:54 smithi151 ceph-mon[155575]: pgmap v96: 129 pgs: 25 active+undersized+degraded, 11 active+undersized, 93 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 126/1389 objects degraded (9.071%) 2023-11-16T13:41:55.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:54 smithi167 ceph-mon[138945]: osdmap e87: 8 total, 8 up, 8 in 2023-11-16T13:41:55.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:54 smithi167 ceph-mon[138945]: pgmap v96: 129 pgs: 25 active+undersized+degraded, 11 active+undersized, 93 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 126/1389 objects degraded (9.071%) 2023-11-16T13:41:56.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:55 smithi151 ceph-mon[155575]: Health check update: Degraded data redundancy: 126/1389 objects degraded (9.071%), 25 pgs degraded (PG_DEGRADED) 2023-11-16T13:41:56.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:55 smithi167 ceph-mon[138945]: Health check update: Degraded data redundancy: 126/1389 objects degraded (9.071%), 25 pgs degraded (PG_DEGRADED) 2023-11-16T13:41:57.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:57 smithi167 ceph-mon[138945]: pgmap v97: 129 pgs: 17 active+undersized+degraded, 8 active+undersized, 104 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 75/1389 objects degraded (5.400%) 2023-11-16T13:41:57.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:57 smithi151 ceph-mon[155575]: pgmap v97: 129 pgs: 17 active+undersized+degraded, 8 active+undersized, 104 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 75/1389 objects degraded (5.400%) 2023-11-16T13:41:57.915 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:41:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:59 smithi167 ceph-mon[138945]: pgmap v98: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 336 B/s wr, 10 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:41:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:59 smithi167 ceph-mon[138945]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 75/1389 objects degraded (5.400%), 17 pgs degraded) 2023-11-16T13:41:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:41:59 smithi167 ceph-mon[138945]: Cluster is now healthy 2023-11-16T13:41:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:59 smithi151 ceph-mon[155575]: pgmap v98: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 336 B/s wr, 10 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:41:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:41:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:41:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:59 smithi151 ceph-mon[155575]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 75/1389 objects degraded (5.400%), 17 pgs degraded) 2023-11-16T13:41:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:41:59 smithi151 ceph-mon[155575]: Cluster is now healthy 2023-11-16T13:42:01.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:01 smithi167 ceph-mon[138945]: pgmap v99: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 255 B/s wr, 7 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:01.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:01 smithi151 ceph-mon[155575]: pgmap v99: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 255 B/s wr, 7 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:02.954 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:42:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:03 smithi151 ceph-mon[155575]: pgmap v100: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 112 B/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:03.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:03 smithi167 ceph-mon[138945]: pgmap v100: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 112 B/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:05.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:05 smithi151 ceph-mon[155575]: pgmap v101: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 203 B/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:05.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:05 smithi167 ceph-mon[138945]: pgmap v101: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 203 B/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:07.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:07 smithi167 ceph-mon[138945]: pgmap v102: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 170 B/s wr, 5 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:07.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:07 smithi151 ceph-mon[155575]: pgmap v102: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 170 B/s wr, 5 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:08.000 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:42:09.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:09 smithi167 ceph-mon[138945]: pgmap v103: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 255 B/s wr, 5 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:09.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:09 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-11-16T13:42:09.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:09 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-11-16T13:42:09.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:09 smithi167 ceph-mon[138945]: Upgrade: osd.4 is safe to restart 2023-11-16T13:42:09.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:09 smithi151 ceph-mon[155575]: pgmap v103: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 255 B/s wr, 5 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:09.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:09 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-11-16T13:42:09.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:09 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-11-16T13:42:09.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:09 smithi151 ceph-mon[155575]: Upgrade: osd.4 is safe to restart 2023-11-16T13:42:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:10 smithi151 ceph-mon[155575]: Upgrade: Updating osd.4 2023-11-16T13:42:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:10 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:10 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-11-16T13:42:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:10 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:42:10.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:10 smithi151 ceph-mon[155575]: Deploying daemon osd.4 on smithi167 2023-11-16T13:42:10.842 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:10 smithi167 ceph-mon[138945]: Upgrade: Updating osd.4 2023-11-16T13:42:10.842 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:10 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:10.842 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:10 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-11-16T13:42:10.843 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:10 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:42:10.843 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:10 smithi167 ceph-mon[138945]: Deploying daemon osd.4 on smithi167 2023-11-16T13:42:11.419 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:11 smithi167 systemd[1]: Stopping Ceph osd.4 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:42:11.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:11 smithi151 ceph-mon[155575]: pgmap v104: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:42:11.780 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:11 smithi167 ceph-mon[138945]: pgmap v104: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:42:11.780 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.4[115342]: debug 2023-11-16T13:42:11.469+0000 7f2523da9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-11-16T13:42:11.780 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.4[115342]: debug 2023-11-16T13:42:11.469+0000 7f2523da9700 -1 osd.4 87 *** Got signal Terminated *** 2023-11-16T13:42:11.780 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.4[115342]: debug 2023-11-16T13:42:11.469+0000 7f2523da9700 -1 osd.4 87 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:42:11.780 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:11 smithi167 bash[144252]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.4 2023-11-16T13:42:12.701 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.3 2023-11-16T13:42:12.701 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 failed (root=default,host=smithi167) (connection refused reported by osd.3) 2023-11-16T13:42:12.701 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.1 2023-11-16T13:42:12.701 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.2 2023-11-16T13:42:12.701 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.3 2023-11-16T13:42:12.701 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.7 2023-11-16T13:42:12.701 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.5 2023-11-16T13:42:12.702 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.6 2023-11-16T13:42:12.702 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.0 2023-11-16T13:42:12.702 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.1 2023-11-16T13:42:12.702 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.2 2023-11-16T13:42:12.702 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.7 2023-11-16T13:42:12.702 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.5 2023-11-16T13:42:12.702 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.6 2023-11-16T13:42:12.702 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.0 2023-11-16T13:42:12.702 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.3 2023-11-16T13:42:12.702 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.3 2023-11-16T13:42:12.702 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.0 2023-11-16T13:42:12.702 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.1 2023-11-16T13:42:12.703 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.2 2023-11-16T13:42:12.703 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.7 2023-11-16T13:42:12.703 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.5 2023-11-16T13:42:12.703 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.6 2023-11-16T13:42:12.703 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.0 2023-11-16T13:42:12.703 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.1 2023-11-16T13:42:12.703 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.2 2023-11-16T13:42:12.703 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.7 2023-11-16T13:42:12.703 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.5 2023-11-16T13:42:12.703 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:12 smithi167 ceph-mon[138945]: osd.4 reported immediately failed by osd.6 2023-11-16T13:42:12.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.3 2023-11-16T13:42:12.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 failed (root=default,host=smithi167) (connection refused reported by osd.3) 2023-11-16T13:42:12.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.1 2023-11-16T13:42:12.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.2 2023-11-16T13:42:12.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.3 2023-11-16T13:42:12.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.7 2023-11-16T13:42:12.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.5 2023-11-16T13:42:12.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.6 2023-11-16T13:42:12.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.0 2023-11-16T13:42:12.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.1 2023-11-16T13:42:12.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.2 2023-11-16T13:42:12.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.7 2023-11-16T13:42:12.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.5 2023-11-16T13:42:12.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.6 2023-11-16T13:42:12.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.0 2023-11-16T13:42:12.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.3 2023-11-16T13:42:12.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.3 2023-11-16T13:42:12.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.0 2023-11-16T13:42:12.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.1 2023-11-16T13:42:12.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.2 2023-11-16T13:42:12.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.7 2023-11-16T13:42:12.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.5 2023-11-16T13:42:12.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.6 2023-11-16T13:42:12.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.0 2023-11-16T13:42:12.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.1 2023-11-16T13:42:12.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.2 2023-11-16T13:42:12.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.7 2023-11-16T13:42:12.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.5 2023-11-16T13:42:12.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:12 smithi151 ceph-mon[155575]: osd.4 reported immediately failed by osd.6 2023-11-16T13:42:13.298 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:12 smithi167 systemd[1]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.4.service: Succeeded. 2023-11-16T13:42:13.298 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:12 smithi167 systemd[1]: Stopped Ceph osd.4 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:42:13.298 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:13 smithi167 systemd[1]: Starting Ceph osd.4 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:42:13.489 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:42:13.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:13 smithi167 ceph-mon[138945]: pgmap v105: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:42:13.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:13 smithi167 ceph-mon[138945]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:42:13.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:13 smithi167 ceph-mon[138945]: osdmap e88: 8 total, 7 up, 8 in 2023-11-16T13:42:13.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:13.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:42:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:13 smithi151 ceph-mon[155575]: pgmap v105: 129 pgs: 129 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:42:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:13 smithi151 ceph-mon[155575]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:42:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:13 smithi151 ceph-mon[155575]: osdmap e88: 8 total, 7 up, 8 in 2023-11-16T13:42:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:42:14.096 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:13 smithi167 bash[144531]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-11-16T13:42:14.097 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:13 smithi167 bash[144531]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2023-11-16T13:42:14.097 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:13 smithi167 bash[144531]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2023-11-16T13:42:14.097 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:13 smithi167 bash[144531]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-11-16T13:42:14.097 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:13 smithi167 bash[144531]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2023-11-16T13:42:14.097 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:13 smithi167 bash[144531]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-11-16T13:42:14.097 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:13 smithi167 bash[144531]: --> ceph-volume raw activate successful for osd ID: 4 2023-11-16T13:42:14.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:14 smithi151 ceph-mon[155575]: osdmap e89: 8 total, 7 up, 8 in 2023-11-16T13:42:14.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:14 smithi167 ceph-mon[138945]: osdmap e89: 8 total, 7 up, 8 in 2023-11-16T13:42:14.847 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:14 smithi167 ceph-osd[144846]: -- 172.21.15.167:0/3438253524 <== mon.1 v2:172.21.15.167:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55edbddf0b60 con 0x55edbddb6c00 2023-11-16T13:42:14.847 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:14 smithi167 bash[144808]: cbb1a9fdea87269763bd72c4592ed1ae88f85ce3c6a42aa93ca7eebe4a125768 2023-11-16T13:42:14.847 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:14 smithi167 systemd[1]: Started Ceph osd.4 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:42:15.346 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-4[144822]: 2023-11-16T13:42:15.059+0000 7f1bd180d540 -1 Falling back to public interface 2023-11-16T13:42:15.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:15 smithi151 ceph-mon[155575]: pgmap v108: 129 pgs: 16 active+undersized+degraded, 4 peering, 9 stale+active+clean, 11 active+undersized, 89 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 122/1389 objects degraded (8.783%) 2023-11-16T13:42:15.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:15 smithi151 ceph-mon[155575]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-11-16T13:42:15.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:15 smithi151 ceph-mon[155575]: Health check failed: Degraded data redundancy: 122/1389 objects degraded (8.783%), 16 pgs degraded (PG_DEGRADED) 2023-11-16T13:42:15.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:15.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:15.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:42:15.780 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:15 smithi167 ceph-mon[138945]: pgmap v108: 129 pgs: 16 active+undersized+degraded, 4 peering, 9 stale+active+clean, 11 active+undersized, 89 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 122/1389 objects degraded (8.783%) 2023-11-16T13:42:15.780 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:15 smithi167 ceph-mon[138945]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-11-16T13:42:15.780 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:15 smithi167 ceph-mon[138945]: Health check failed: Degraded data redundancy: 122/1389 objects degraded (8.783%), 16 pgs degraded (PG_DEGRADED) 2023-11-16T13:42:15.780 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:15.780 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:15.780 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:42:17.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:17 smithi151 ceph-mon[155575]: pgmap v109: 129 pgs: 28 active+undersized+degraded, 4 peering, 5 stale+active+clean, 16 active+undersized, 76 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 203/1389 objects degraded (14.615%) 2023-11-16T13:42:17.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:17 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:17.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:17 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:17.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:17 smithi167 ceph-mon[138945]: pgmap v109: 129 pgs: 28 active+undersized+degraded, 4 peering, 5 stale+active+clean, 16 active+undersized, 76 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 203/1389 objects degraded (14.615%) 2023-11-16T13:42:17.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:17 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:17.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:17 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:18.523 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:42:18.911 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:18.911 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:18.911 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:18 smithi167 ceph-mon[138945]: pgmap v110: 129 pgs: 32 active+undersized+degraded, 24 active+undersized, 73 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 226/1389 objects degraded (16.271%) 2023-11-16T13:42:18.945 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:42:18.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:18.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:18.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:18 smithi151 ceph-mon[155575]: pgmap v110: 129 pgs: 32 active+undersized+degraded, 24 active+undersized, 73 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 226/1389 objects degraded (16.271%) 2023-11-16T13:42:19.555 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:42:19.555 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (10m) 29s ago 11m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:42:19.555 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (2m) 29s ago 11m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:42:19.555 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (2m) 2s ago 10m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:42:19.555 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (10m) 29s ago 11m 36.4M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:42:19.555 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (7m) 29s ago 7m 16.3M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:42:19.555 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (7m) 2s ago 7m 42.3M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:42:19.556 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (3m) 29s ago 12m 501M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:42:19.556 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (3m) 2s ago 10m 439M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:42:19.556 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (2m) 29s ago 12m 91.6M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:42:19.556 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (2m) 2s ago 10m 80.8M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:42:19.556 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (5m) 29s ago 5m 85.0M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:42:19.556 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (5m) 2s ago 5m 56.6M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:42:19.556 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (11m) 29s ago 11m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:42:19.556 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (10m) 2s ago 10m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:42:19.556 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (119s) 29s ago 10m 382M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:42:19.556 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (90s) 29s ago 10m 291M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 49b15d0ef2d2 2023-11-16T13:42:19.556 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (61s) 29s ago 9m 216M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 89fcacc46719 2023-11-16T13:42:19.556 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (33s) 29s ago 9m 12.4M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 aa3cf87e766a 2023-11-16T13:42:19.556 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (5s) 2s ago 9m 14.3M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 cbb1a9fdea87 2023-11-16T13:42:19.557 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (9m) 2s ago 9m 1386M 2712M 16.2.5 6933c2a0b7dd 131fb5b5d574 2023-11-16T13:42:19.557 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (9m) 2s ago 9m 1067M 2712M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:42:19.557 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (8m) 2s ago 8m 1838M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:42:19.557 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (3m) 29s ago 10m 64.3M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:42:19.816 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-4[144822]: 2023-11-16T13:42:19.507+0000 7f1bd180d540 -1 osd.4 87 log_to_monitors true 2023-11-16T13:42:19.919 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:42:19.919 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:42:19.919 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:42:19.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:42:19.919 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:42:19.919 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:42:19.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:42:19.920 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:42:19.920 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 3, 2023-11-16T13:42:19.920 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 4 2023-11-16T13:42:19.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:42:19.920 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:42:19.920 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:42:19.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:42:19.920 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:42:19.920 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 5, 2023-11-16T13:42:19.920 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 8 2023-11-16T13:42:19.920 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:42:19.920 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:42:20.077 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:42:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-4[144822]: 2023-11-16T13:42:19.843+0000 7f1bc3def700 -1 osd.4 87 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-11-16T13:42:20.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:19 smithi167 ceph-mon[138945]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-11-16T13:42:20.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:19 smithi167 ceph-mon[138945]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:20.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:19 smithi167 ceph-mon[138945]: from='client.34250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:20.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:19 smithi167 ceph-mon[138945]: from='osd.4 [v2:172.21.15.167:6800/2650371046,v1:172.21.15.167:6801/2650371046]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-11-16T13:42:20.078 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:19 smithi167 ceph-mon[138945]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-11-16T13:42:20.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:19 smithi151 ceph-mon[155575]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-11-16T13:42:20.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:19 smithi151 ceph-mon[155575]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:20.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:19 smithi151 ceph-mon[155575]: from='client.34250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:20.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:19 smithi151 ceph-mon[155575]: from='osd.4 [v2:172.21.15.167:6800/2650371046,v1:172.21.15.167:6801/2650371046]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-11-16T13:42:20.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:19 smithi151 ceph-mon[155575]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-11-16T13:42:20.244 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:42:20.244 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:42:20.244 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:42:20.245 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:42:20.245 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:42:20.245 INFO:teuthology.orchestra.run.smithi151.stdout: "crash", 2023-11-16T13:42:20.245 INFO:teuthology.orchestra.run.smithi151.stdout: "mon", 2023-11-16T13:42:20.245 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:42:20.245 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:42:20.246 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "11/23 daemons upgraded", 2023-11-16T13:42:20.246 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading osd daemons", 2023-11-16T13:42:20.246 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:42:20.246 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:42:21.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:21.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-11-16T13:42:21.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: osdmap e90: 8 total, 7 up, 8 in 2023-11-16T13:42:21.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='osd.4 [v2:172.21.15.167:6800/2650371046,v1:172.21.15.167:6801/2650371046]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:42:21.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:42:21.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/3991469349' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:21.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: pgmap v112: 129 pgs: 32 active+undersized+degraded, 24 active+undersized, 73 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 270 B/s wr, 0 op/s; 226/1389 objects degraded (16.271%) 2023-11-16T13:42:21.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:21.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:21.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:42:21.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:42:21.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:21.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:21.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:21.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:42:21.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:21.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:21.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:21.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2023-11-16T13:42:21.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2023-11-16T13:42:21.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:20 smithi167 ceph-mon[138945]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-11-16T13:42:21.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:21.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-11-16T13:42:21.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: osdmap e90: 8 total, 7 up, 8 in 2023-11-16T13:42:21.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='osd.4 [v2:172.21.15.167:6800/2650371046,v1:172.21.15.167:6801/2650371046]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:42:21.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:42:21.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/3991469349' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:21.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: pgmap v112: 129 pgs: 32 active+undersized+degraded, 24 active+undersized, 73 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 270 B/s wr, 0 op/s; 226/1389 objects degraded (16.271%) 2023-11-16T13:42:21.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:21.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:21.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:42:21.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:42:21.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:21.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:21.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:21.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:42:21.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:21.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:21.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:21.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2023-11-16T13:42:21.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2023-11-16T13:42:21.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:20 smithi151 ceph-mon[155575]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-11-16T13:42:22.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:21 smithi167 ceph-mon[138945]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:42:22.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:21 smithi167 ceph-mon[138945]: osd.4 [v2:172.21.15.167:6800/2650371046,v1:172.21.15.167:6801/2650371046] boot 2023-11-16T13:42:22.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:21 smithi167 ceph-mon[138945]: osdmap e91: 8 total, 8 up, 8 in 2023-11-16T13:42:22.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:21 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:42:22.099 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:21 smithi167 ceph-mon[138945]: Health check update: Degraded data redundancy: 226/1389 objects degraded (16.271%), 32 pgs degraded (PG_DEGRADED) 2023-11-16T13:42:22.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:21 smithi151 ceph-mon[155575]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:42:22.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:21 smithi151 ceph-mon[155575]: osd.4 [v2:172.21.15.167:6800/2650371046,v1:172.21.15.167:6801/2650371046] boot 2023-11-16T13:42:22.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:21 smithi151 ceph-mon[155575]: osdmap e91: 8 total, 8 up, 8 in 2023-11-16T13:42:22.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:21 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-11-16T13:42:22.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:21 smithi151 ceph-mon[155575]: Health check update: Degraded data redundancy: 226/1389 objects degraded (16.271%), 32 pgs degraded (PG_DEGRADED) 2023-11-16T13:42:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:22 smithi151 ceph-mon[155575]: osdmap e92: 8 total, 8 up, 8 in 2023-11-16T13:42:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:22 smithi151 ceph-mon[155575]: pgmap v115: 129 pgs: 30 active+undersized+degraded, 23 active+undersized, 76 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 213/1389 objects degraded (15.335%) 2023-11-16T13:42:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:22 smithi167 ceph-mon[138945]: osdmap e92: 8 total, 8 up, 8 in 2023-11-16T13:42:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:22 smithi167 ceph-mon[138945]: pgmap v115: 129 pgs: 30 active+undersized+degraded, 23 active+undersized, 76 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 213/1389 objects degraded (15.335%) 2023-11-16T13:42:23.565 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:42:25.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:25 smithi167 ceph-mon[138945]: pgmap v116: 129 pgs: 15 active+undersized+degraded, 16 active+undersized, 98 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s; 102/1389 objects degraded (7.343%); 0 B/s, 0 objects/s recovering 2023-11-16T13:42:25.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:25 smithi151 ceph-mon[155575]: pgmap v116: 129 pgs: 15 active+undersized+degraded, 16 active+undersized, 98 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s; 102/1389 objects degraded (7.343%); 0 B/s, 0 objects/s recovering 2023-11-16T13:42:26.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:26 smithi151 ceph-mon[155575]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 102/1389 objects degraded (7.343%), 15 pgs degraded) 2023-11-16T13:42:26.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:26 smithi151 ceph-mon[155575]: Cluster is now healthy 2023-11-16T13:42:26.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:26 smithi167 ceph-mon[138945]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 102/1389 objects degraded (7.343%), 15 pgs degraded) 2023-11-16T13:42:26.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:26 smithi167 ceph-mon[138945]: Cluster is now healthy 2023-11-16T13:42:27.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:27 smithi167 ceph-mon[138945]: pgmap v117: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 165 B/s rd, 165 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:27 smithi151 ceph-mon[155575]: pgmap v117: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 165 B/s rd, 165 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:28.605 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:42:29.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:29 smithi167 ceph-mon[138945]: pgmap v118: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:29 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:29 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:42:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:29 smithi151 ceph-mon[155575]: pgmap v118: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:29 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:29 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:42:31.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:31 smithi167 ceph-mon[138945]: pgmap v119: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 223 B/s rd, 223 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:31.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:31 smithi151 ceph-mon[155575]: pgmap v119: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 223 B/s rd, 223 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:33.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:33 smithi151 ceph-mon[155575]: pgmap v120: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 302 B/s rd, 201 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:33.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:33 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:33.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:33 smithi167 ceph-mon[138945]: pgmap v120: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 302 B/s rd, 201 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:33.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:33 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:33.648 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:42:35.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:35 smithi151 ceph-mon[155575]: pgmap v121: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:35.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:35 smithi167 ceph-mon[138945]: pgmap v121: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-11-16T13:42:36.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:36 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2023-11-16T13:42:36.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:36 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2023-11-16T13:42:36.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:36 smithi151 ceph-mon[155575]: Upgrade: osd.5 is safe to restart 2023-11-16T13:42:36.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:36 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2023-11-16T13:42:36.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:36 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2023-11-16T13:42:36.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:36 smithi167 ceph-mon[138945]: Upgrade: osd.5 is safe to restart 2023-11-16T13:42:37.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:37 smithi167 ceph-mon[138945]: pgmap v122: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:42:37.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:37 smithi167 ceph-mon[138945]: Upgrade: Updating osd.5 2023-11-16T13:42:37.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:37 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:37.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:37 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-11-16T13:42:37.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:37 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:42:37.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:37 smithi167 ceph-mon[138945]: Deploying daemon osd.5 on smithi167 2023-11-16T13:42:37.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:37 smithi151 ceph-mon[155575]: pgmap v122: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:42:37.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:37 smithi151 ceph-mon[155575]: Upgrade: Updating osd.5 2023-11-16T13:42:37.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:37 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:37.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:37 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-11-16T13:42:37.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:37 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:42:37.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:37 smithi151 ceph-mon[155575]: Deploying daemon osd.5 on smithi167 2023-11-16T13:42:38.346 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:38 smithi167 systemd[1]: Stopping Ceph osd.5 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:42:38.347 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.5[118177]: debug 2023-11-16T13:42:38.302+0000 7f0822e03700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-11-16T13:42:38.347 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.5[118177]: debug 2023-11-16T13:42:38.302+0000 7f0822e03700 -1 osd.5 92 *** Got signal Terminated *** 2023-11-16T13:42:38.347 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:38 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.5[118177]: debug 2023-11-16T13:42:38.302+0000 7f0822e03700 -1 osd.5 92 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:42:38.699 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:42:38.846 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:38 smithi167 bash[146981]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.5 2023-11-16T13:42:39.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: pgmap v123: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:42:39.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.7 2023-11-16T13:42:39.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 failed (root=default,host=smithi167) (connection refused reported by osd.7) 2023-11-16T13:42:39.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.7 2023-11-16T13:42:39.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.6 2023-11-16T13:42:39.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.4 2023-11-16T13:42:39.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.6 2023-11-16T13:42:39.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.0 2023-11-16T13:42:39.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.1 2023-11-16T13:42:39.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.2 2023-11-16T13:42:39.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.3 2023-11-16T13:42:39.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.4 2023-11-16T13:42:39.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.0 2023-11-16T13:42:39.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.1 2023-11-16T13:42:39.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.2 2023-11-16T13:42:39.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.3 2023-11-16T13:42:39.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.6 2023-11-16T13:42:39.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.6 2023-11-16T13:42:39.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.7 2023-11-16T13:42:39.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.4 2023-11-16T13:42:39.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.7 2023-11-16T13:42:39.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.4 2023-11-16T13:42:39.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.2 2023-11-16T13:42:39.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.3 2023-11-16T13:42:39.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.0 2023-11-16T13:42:39.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.1 2023-11-16T13:42:39.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.2 2023-11-16T13:42:39.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.3 2023-11-16T13:42:39.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.0 2023-11-16T13:42:39.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:39 smithi167 ceph-mon[138945]: osd.5 reported immediately failed by osd.1 2023-11-16T13:42:39.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: pgmap v123: 129 pgs: 129 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:42:39.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.7 2023-11-16T13:42:39.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 failed (root=default,host=smithi167) (connection refused reported by osd.7) 2023-11-16T13:42:39.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.7 2023-11-16T13:42:39.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.6 2023-11-16T13:42:39.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.4 2023-11-16T13:42:39.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.6 2023-11-16T13:42:39.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.0 2023-11-16T13:42:39.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.1 2023-11-16T13:42:39.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.2 2023-11-16T13:42:39.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.3 2023-11-16T13:42:39.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.4 2023-11-16T13:42:39.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.0 2023-11-16T13:42:39.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.1 2023-11-16T13:42:39.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.2 2023-11-16T13:42:39.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.3 2023-11-16T13:42:39.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.6 2023-11-16T13:42:39.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.6 2023-11-16T13:42:39.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.7 2023-11-16T13:42:39.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.4 2023-11-16T13:42:39.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.7 2023-11-16T13:42:39.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.4 2023-11-16T13:42:39.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.2 2023-11-16T13:42:39.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.3 2023-11-16T13:42:39.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.0 2023-11-16T13:42:39.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.1 2023-11-16T13:42:39.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.2 2023-11-16T13:42:39.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.3 2023-11-16T13:42:39.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.0 2023-11-16T13:42:39.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:39 smithi151 ceph-mon[155575]: osd.5 reported immediately failed by osd.1 2023-11-16T13:42:39.939 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:39 smithi167 systemd[1]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.5.service: Succeeded. 2023-11-16T13:42:39.940 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:39 smithi167 systemd[1]: Stopped Ceph osd.5 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:42:40.202 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:39 smithi167 systemd[1]: Starting Ceph osd.5 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:42:40.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:40 smithi167 ceph-mon[138945]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:42:40.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:40 smithi167 ceph-mon[138945]: osdmap e93: 8 total, 7 up, 8 in 2023-11-16T13:42:40.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:40 smithi151 ceph-mon[155575]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:42:40.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:40 smithi151 ceph-mon[155575]: osdmap e93: 8 total, 7 up, 8 in 2023-11-16T13:42:41.097 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:40 smithi167 bash[147262]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-11-16T13:42:41.097 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:40 smithi167 bash[147262]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2023-11-16T13:42:41.097 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:40 smithi167 bash[147262]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2023-11-16T13:42:41.097 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:40 smithi167 bash[147262]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-11-16T13:42:41.097 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:40 smithi167 bash[147262]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2023-11-16T13:42:41.097 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:40 smithi167 bash[147262]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-11-16T13:42:41.097 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:40 smithi167 bash[147262]: --> ceph-volume raw activate successful for osd ID: 5 2023-11-16T13:42:41.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:41 smithi167 ceph-mon[138945]: pgmap v125: 129 pgs: 6 peering, 14 stale+active+clean, 109 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:42:41.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:41 smithi167 ceph-mon[138945]: Health check failed: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2023-11-16T13:42:41.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:41 smithi167 ceph-mon[138945]: osdmap e94: 8 total, 7 up, 8 in 2023-11-16T13:42:41.597 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:41 smithi167 ceph-osd[147570]: -- 172.21.15.167:0/2082564764 <== mon.1 v2:172.21.15.167:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b21bdd8b60 con 0x55b21bd9ec00 2023-11-16T13:42:41.597 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:41 smithi167 bash[147533]: 82902ee5659d8bd2f5ec1b59895e5411c345e23cdef24e6ba77ef5501bac073d 2023-11-16T13:42:41.597 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:41 smithi167 systemd[1]: Started Ceph osd.5 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:42:41.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:41 smithi151 ceph-mon[155575]: pgmap v125: 129 pgs: 6 peering, 14 stale+active+clean, 109 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:42:41.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:41 smithi151 ceph-mon[155575]: Health check failed: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2023-11-16T13:42:41.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:41 smithi151 ceph-mon[155575]: osdmap e94: 8 total, 7 up, 8 in 2023-11-16T13:42:42.347 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:41 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:42:41.846+0000 7fec33ed1540 -1 Falling back to public interface 2023-11-16T13:42:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:42.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:42:42.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:42.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:42.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:42:43.704 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:43 smithi167 ceph-mon[138945]: pgmap v127: 129 pgs: 5 active+undersized+degraded, 11 peering, 9 stale+active+clean, 1 active+undersized, 103 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 33/1389 objects degraded (2.376%) 2023-11-16T13:42:43.705 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:43 smithi167 ceph-mon[138945]: Health check failed: Degraded data redundancy: 33/1389 objects degraded (2.376%), 5 pgs degraded (PG_DEGRADED) 2023-11-16T13:42:43.705 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:43.705 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:42:43.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:43 smithi151 ceph-mon[155575]: pgmap v127: 129 pgs: 5 active+undersized+degraded, 11 peering, 9 stale+active+clean, 1 active+undersized, 103 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 33/1389 objects degraded (2.376%) 2023-11-16T13:42:43.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:43 smithi151 ceph-mon[155575]: Health check failed: Degraded data redundancy: 33/1389 objects degraded (2.376%), 5 pgs degraded (PG_DEGRADED) 2023-11-16T13:42:43.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:43.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:42:43.735 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:42:45.212 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:44 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:45.212 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:44 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:45.212 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:44 smithi167 ceph-mon[138945]: pgmap v128: 129 pgs: 15 active+undersized+degraded, 11 peering, 3 stale+active+clean, 10 active+undersized, 90 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 94/1389 objects degraded (6.767%) 2023-11-16T13:42:45.212 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:44 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:45.213 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:44 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:45.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:44 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:45.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:44 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:45.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:44 smithi151 ceph-mon[155575]: pgmap v128: 129 pgs: 15 active+undersized+degraded, 11 peering, 3 stale+active+clean, 10 active+undersized, 90 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 94/1389 objects degraded (6.767%) 2023-11-16T13:42:45.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:44 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:45.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:44 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:46.466 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:46 smithi167 ceph-mon[138945]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 5 pgs peering) 2023-11-16T13:42:46.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:46 smithi151 ceph-mon[155575]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 5 pgs peering) 2023-11-16T13:42:46.804 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:46 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:42:46.462+0000 7fec33ed1540 -1 osd.5 92 log_to_monitors true 2023-11-16T13:42:47.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: pgmap v129: 129 pgs: 24 active+undersized+degraded, 21 active+undersized, 84 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 154/1389 objects degraded (11.087%) 2023-11-16T13:42:47.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: from='osd.5 [v2:172.21.15.167:6808/4210380934,v1:172.21.15.167:6809/4210380934]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-11-16T13:42:47.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-11-16T13:42:47.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:47.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:47.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:42:47.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:42:47.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:47.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:47.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:42:47.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:47.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:47.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:47.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2023-11-16T13:42:47.596 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:42:47.221+0000 7fec24cb0700 -1 osd.5 92 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-11-16T13:42:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: pgmap v129: 129 pgs: 24 active+undersized+degraded, 21 active+undersized, 84 active+clean; 318 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 154/1389 objects degraded (11.087%) 2023-11-16T13:42:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: from='osd.5 [v2:172.21.15.167:6808/4210380934,v1:172.21.15.167:6809/4210380934]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-11-16T13:42:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-11-16T13:42:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:42:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:42:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:47.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:42:47.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:47.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:47.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:47.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2023-11-16T13:42:48.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:48 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2023-11-16T13:42:48.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:48 smithi151 ceph-mon[155575]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2023-11-16T13:42:48.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:48 smithi151 ceph-mon[155575]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-11-16T13:42:48.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:48 smithi151 ceph-mon[155575]: osdmap e95: 8 total, 7 up, 8 in 2023-11-16T13:42:48.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:48 smithi151 ceph-mon[155575]: from='osd.5 [v2:172.21.15.167:6808/4210380934,v1:172.21.15.167:6809/4210380934]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:42:48.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:48 smithi151 ceph-mon[155575]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:42:48.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:48 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2023-11-16T13:42:48.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:48 smithi167 ceph-mon[138945]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2023-11-16T13:42:48.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:48 smithi167 ceph-mon[138945]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-11-16T13:42:48.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:48 smithi167 ceph-mon[138945]: osdmap e95: 8 total, 7 up, 8 in 2023-11-16T13:42:48.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:48 smithi167 ceph-mon[138945]: from='osd.5 [v2:172.21.15.167:6808/4210380934,v1:172.21.15.167:6809/4210380934]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:42:48.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:48 smithi167 ceph-mon[138945]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:42:48.767 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:42:49.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:49 smithi151 ceph-mon[155575]: pgmap v131: 129 pgs: 24 active+undersized+degraded, 21 active+undersized, 84 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 154/1389 objects degraded (11.087%) 2023-11-16T13:42:49.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:49 smithi151 ceph-mon[155575]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:42:49.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:49 smithi151 ceph-mon[155575]: osd.5 [v2:172.21.15.167:6808/4210380934,v1:172.21.15.167:6809/4210380934] boot 2023-11-16T13:42:49.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:49 smithi151 ceph-mon[155575]: osdmap e96: 8 total, 8 up, 8 in 2023-11-16T13:42:49.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:49 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:42:49.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:49 smithi151 ceph-mon[155575]: osdmap e97: 8 total, 8 up, 8 in 2023-11-16T13:42:49.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:49 smithi167 ceph-mon[138945]: pgmap v131: 129 pgs: 24 active+undersized+degraded, 21 active+undersized, 84 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 154/1389 objects degraded (11.087%) 2023-11-16T13:42:49.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:49 smithi167 ceph-mon[138945]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:42:49.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:49 smithi167 ceph-mon[138945]: osd.5 [v2:172.21.15.167:6808/4210380934,v1:172.21.15.167:6809/4210380934] boot 2023-11-16T13:42:49.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:49 smithi167 ceph-mon[138945]: osdmap e96: 8 total, 8 up, 8 in 2023-11-16T13:42:49.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:49 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-11-16T13:42:49.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:49 smithi167 ceph-mon[138945]: osdmap e97: 8 total, 8 up, 8 in 2023-11-16T13:42:50.577 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:42:50.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:50 smithi167 ceph-mon[138945]: Health check update: Degraded data redundancy: 96/1389 objects degraded (6.911%), 15 pgs degraded (PG_DEGRADED) 2023-11-16T13:42:50.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:50 smithi151 ceph-mon[155575]: Health check update: Degraded data redundancy: 96/1389 objects degraded (6.911%), 15 pgs degraded (PG_DEGRADED) 2023-11-16T13:42:51.184 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:42:51.184 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (10m) 61s ago 12m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:42:51.184 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (2m) 61s ago 12m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:42:51.185 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (2m) 7s ago 11m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:42:51.185 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (10m) 61s ago 11m 36.4M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:42:51.185 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (8m) 61s ago 8m 16.3M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:42:51.185 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (8m) 7s ago 8m 42.3M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:42:51.185 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (4m) 61s ago 13m 501M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:42:51.185 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (3m) 7s ago 11m 439M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:42:51.185 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (3m) 61s ago 13m 91.6M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:42:51.185 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (3m) 7s ago 11m 83.5M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:42:51.185 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (6m) 61s ago 6m 85.0M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:42:51.185 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (6m) 7s ago 6m 68.7M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:42:51.185 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (11m) 61s ago 11m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:42:51.185 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (11m) 7s ago 11m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:42:51.186 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (2m) 61s ago 10m 382M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:42:51.186 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (2m) 61s ago 10m 291M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 49b15d0ef2d2 2023-11-16T13:42:51.186 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (93s) 61s ago 10m 216M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 89fcacc46719 2023-11-16T13:42:51.186 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (65s) 61s ago 10m 12.4M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 aa3cf87e766a 2023-11-16T13:42:51.186 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (36s) 7s ago 9m 382M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 cbb1a9fdea87 2023-11-16T13:42:51.186 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (10s) 7s ago 9m 14.2M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 82902ee5659d 2023-11-16T13:42:51.186 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (9m) 7s ago 9m 1073M 2712M 16.2.5 6933c2a0b7dd 137468c24f14 2023-11-16T13:42:51.186 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (9m) 7s ago 9m 1845M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:42:51.186 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (4m) 61s ago 11m 64.3M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:42:51.549 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:42:51.549 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:42:51.549 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:42:51.550 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:42:51.550 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:42:51.550 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:42:51.550 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:42:51.550 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:42:51.551 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2, 2023-11-16T13:42:51.551 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 6 2023-11-16T13:42:51.551 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:42:51.551 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:42:51.551 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:42:51.552 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:42:51.552 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:42:51.552 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 4, 2023-11-16T13:42:51.552 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 10 2023-11-16T13:42:51.552 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:42:51.552 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:42:51.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:51 smithi167 ceph-mon[138945]: pgmap v134: 129 pgs: 9 peering, 15 active+undersized+degraded, 13 active+undersized, 92 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s; 96/1389 objects degraded (6.911%) 2023-11-16T13:42:51.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:51 smithi151 ceph-mon[155575]: pgmap v134: 129 pgs: 9 peering, 15 active+undersized+degraded, 13 active+undersized, 92 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s; 96/1389 objects degraded (6.911%) 2023-11-16T13:42:51.860 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:42:51.860 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:42:51.860 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:42:51.860 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:42:51.861 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:42:51.861 INFO:teuthology.orchestra.run.smithi151.stdout: "crash", 2023-11-16T13:42:51.861 INFO:teuthology.orchestra.run.smithi151.stdout: "mon", 2023-11-16T13:42:51.861 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:42:51.861 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:42:51.861 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "12/23 daemons upgraded", 2023-11-16T13:42:51.861 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading osd daemons", 2023-11-16T13:42:51.862 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:42:51.862 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:42:52.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:52 smithi167 ceph-mon[138945]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:52.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:52 smithi167 ceph-mon[138945]: from='client.34276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:52.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:52 smithi167 ceph-mon[138945]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:52.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:52 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/1169553357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:52.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:52 smithi151 ceph-mon[155575]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:52.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:52 smithi151 ceph-mon[155575]: from='client.34276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:52.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:52 smithi151 ceph-mon[155575]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:52.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:52 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/1169553357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:42:53.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:53 smithi167 ceph-mon[138945]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:53.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:53 smithi167 ceph-mon[138945]: pgmap v135: 129 pgs: 9 peering, 11 active+undersized+degraded, 11 active+undersized, 98 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 62/1389 objects degraded (4.464%) 2023-11-16T13:42:53.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:53 smithi151 ceph-mon[155575]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:42:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:53 smithi151 ceph-mon[155575]: pgmap v135: 129 pgs: 9 peering, 11 active+undersized+degraded, 11 active+undersized, 98 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 62/1389 objects degraded (4.464%) 2023-11-16T13:42:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:53.807 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:42:54.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:54 smithi167 ceph-mon[138945]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 62/1389 objects degraded (4.464%), 11 pgs degraded) 2023-11-16T13:42:54.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:54 smithi167 ceph-mon[138945]: Cluster is now healthy 2023-11-16T13:42:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:54 smithi151 ceph-mon[155575]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 62/1389 objects degraded (4.464%), 11 pgs degraded) 2023-11-16T13:42:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:54 smithi151 ceph-mon[155575]: Cluster is now healthy 2023-11-16T13:42:55.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:55 smithi167 ceph-mon[138945]: pgmap v136: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 150 B/s rd, 300 B/s wr, 0 op/s 2023-11-16T13:42:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:55 smithi151 ceph-mon[155575]: pgmap v136: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 150 B/s rd, 300 B/s wr, 0 op/s 2023-11-16T13:42:57.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:57 smithi167 ceph-mon[138945]: pgmap v137: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 127 B/s rd, 127 B/s wr, 0 op/s 2023-11-16T13:42:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:57 smithi151 ceph-mon[155575]: pgmap v137: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 127 B/s rd, 127 B/s wr, 0 op/s 2023-11-16T13:42:58.844 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:42:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:59 smithi167 ceph-mon[138945]: pgmap v138: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 208 B/s rd, 208 B/s wr, 0 op/s 2023-11-16T13:42:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:42:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:42:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:59 smithi151 ceph-mon[155575]: pgmap v138: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 208 B/s rd, 208 B/s wr, 0 op/s 2023-11-16T13:42:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:42:59.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:42:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:43:01.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:01 smithi167 ceph-mon[138945]: pgmap v139: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 189 B/s rd, 189 B/s wr, 0 op/s 2023-11-16T13:43:01.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:01 smithi151 ceph-mon[155575]: pgmap v139: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 189 B/s rd, 189 B/s wr, 0 op/s 2023-11-16T13:43:02.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2023-11-16T13:43:02.582 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2023-11-16T13:43:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:03 smithi151 ceph-mon[155575]: pgmap v140: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:43:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:03 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2023-11-16T13:43:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:03 smithi151 ceph-mon[155575]: Upgrade: osd.6 is safe to restart 2023-11-16T13:43:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:03 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:03.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:03 smithi167 ceph-mon[138945]: pgmap v140: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:43:03.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:03 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2023-11-16T13:43:03.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:03 smithi167 ceph-mon[138945]: Upgrade: osd.6 is safe to restart 2023-11-16T13:43:03.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:03 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:03.885 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:43:04.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:04 smithi167 ceph-mon[138945]: Upgrade: Updating osd.6 2023-11-16T13:43:04.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:04 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:04.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:04 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-11-16T13:43:04.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:04 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:04.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:04 smithi167 ceph-mon[138945]: Deploying daemon osd.6 on smithi167 2023-11-16T13:43:04.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:04 smithi151 ceph-mon[155575]: Upgrade: Updating osd.6 2023-11-16T13:43:04.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:04 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:04.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:04 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-11-16T13:43:04.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:04 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:04.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:04 smithi151 ceph-mon[155575]: Deploying daemon osd.6 on smithi167 2023-11-16T13:43:05.510 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:05 smithi167 ceph-mon[138945]: pgmap v141: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:43:05.533 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:05 smithi167 systemd[1]: Stopping Ceph osd.6 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:43:05.533 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.6[121036]: debug 2023-11-16T13:43:05.306+0000 7f89b2329700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-11-16T13:43:05.533 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.6[121036]: debug 2023-11-16T13:43:05.306+0000 7f89b2329700 -1 osd.6 97 *** Got signal Terminated *** 2023-11-16T13:43:05.534 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.6[121036]: debug 2023-11-16T13:43:05.306+0000 7f89b2329700 -1 osd.6 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:43:05.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:05 smithi151 ceph-mon[155575]: pgmap v141: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:43:05.846 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:05 smithi167 bash[149674]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.6 2023-11-16T13:43:06.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.7 2023-11-16T13:43:15.109 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 failed (root=default,host=smithi167) (connection refused reported by osd.7) 2023-11-16T13:43:15.146 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.7 2023-11-16T13:43:15.146 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.4 2023-11-16T13:43:15.146 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.3 2023-11-16T13:43:15.146 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.0 2023-11-16T13:43:15.146 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.1 2023-11-16T13:43:15.147 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.2 2023-11-16T13:43:15.147 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.3 2023-11-16T13:43:15.147 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.5 2023-11-16T13:43:15.147 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.0 2023-11-16T13:43:15.147 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.1 2023-11-16T13:43:15.147 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.2 2023-11-16T13:43:15.147 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.5 2023-11-16T13:43:15.147 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.4 2023-11-16T13:43:15.147 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.3 2023-11-16T13:43:15.148 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.7 2023-11-16T13:43:15.148 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.7 2023-11-16T13:43:15.149 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.4 2023-11-16T13:43:15.149 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.1 2023-11-16T13:43:15.149 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.5 2023-11-16T13:43:15.150 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.0 2023-11-16T13:43:15.151 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.1 2023-11-16T13:43:15.151 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.2 2023-11-16T13:43:15.152 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.3 2023-11-16T13:43:15.181 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.4 2023-11-16T13:43:15.182 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.0 2023-11-16T13:43:15.183 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.2 2023-11-16T13:43:15.184 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.3 2023-11-16T13:43:15.185 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.5 2023-11-16T13:43:15.187 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:06 smithi167 ceph-mon[138945]: osd.6 reported immediately failed by osd.3 2023-11-16T13:43:15.192 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:06 smithi167 systemd[1]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.6.service: Succeeded. 2023-11-16T13:43:15.192 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:06 smithi167 systemd[1]: Stopped Ceph osd.6 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:43:15.192 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:06 smithi167 systemd[1]: Starting Ceph osd.6 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:43:15.193 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:07 smithi167 ceph-mon[138945]: pgmap v142: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:43:15.194 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:07 smithi167 ceph-mon[138945]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:43:15.194 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:07 smithi167 ceph-mon[138945]: osdmap e98: 8 total, 7 up, 8 in 2023-11-16T13:43:15.195 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.7 2023-11-16T13:43:15.195 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 failed (root=default,host=smithi167) (connection refused reported by osd.7) 2023-11-16T13:43:15.195 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.7 2023-11-16T13:43:15.196 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.4 2023-11-16T13:43:15.196 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.3 2023-11-16T13:43:15.196 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.0 2023-11-16T13:43:15.196 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.1 2023-11-16T13:43:15.196 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.2 2023-11-16T13:43:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.3 2023-11-16T13:43:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.5 2023-11-16T13:43:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.0 2023-11-16T13:43:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.1 2023-11-16T13:43:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.2 2023-11-16T13:43:15.197 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.5 2023-11-16T13:43:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.4 2023-11-16T13:43:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.3 2023-11-16T13:43:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.7 2023-11-16T13:43:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.7 2023-11-16T13:43:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.4 2023-11-16T13:43:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.1 2023-11-16T13:43:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.5 2023-11-16T13:43:15.198 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.0 2023-11-16T13:43:15.199 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.1 2023-11-16T13:43:15.199 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.2 2023-11-16T13:43:15.199 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.3 2023-11-16T13:43:15.199 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.4 2023-11-16T13:43:15.199 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.0 2023-11-16T13:43:15.199 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.2 2023-11-16T13:43:15.200 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.3 2023-11-16T13:43:15.200 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.5 2023-11-16T13:43:15.200 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:06 smithi151 ceph-mon[155575]: osd.6 reported immediately failed by osd.3 2023-11-16T13:43:15.200 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:07 smithi151 ceph-mon[155575]: pgmap v142: 129 pgs: 129 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:43:15.200 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:07 smithi151 ceph-mon[155575]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:43:15.200 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:07 smithi151 ceph-mon[155575]: osdmap e98: 8 total, 7 up, 8 in 2023-11-16T13:43:15.200 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:08 smithi151 ceph-mon[155575]: osdmap e99: 8 total, 7 up, 8 in 2023-11-16T13:43:15.201 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:43:15.202 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:07 smithi167 bash[149953]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-11-16T13:43:15.202 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:07 smithi167 bash[149953]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2023-11-16T13:43:15.202 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:07 smithi167 bash[149953]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2023-11-16T13:43:15.202 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:07 smithi167 bash[149953]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-11-16T13:43:15.202 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:07 smithi167 bash[149953]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2023-11-16T13:43:15.203 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:07 smithi167 bash[149953]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-11-16T13:43:15.203 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:07 smithi167 bash[149953]: --> ceph-volume raw activate successful for osd ID: 6 2023-11-16T13:43:15.203 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:08 smithi167 ceph-mon[138945]: osdmap e99: 8 total, 7 up, 8 in 2023-11-16T13:43:15.204 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:09 smithi151 ceph-mon[155575]: pgmap v145: 129 pgs: 15 peering, 10 stale+active+clean, 104 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:43:15.204 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:09 smithi151 ceph-mon[155575]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-11-16T13:43:15.204 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:09 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.204 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:09 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.204 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:09 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:43:15.204 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:10 smithi151 ceph-mon[155575]: Health check failed: Degraded data redundancy: 88/1389 objects degraded (6.335%), 15 pgs degraded (PG_DEGRADED) 2023-11-16T13:43:15.204 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:11 smithi151 ceph-mon[155575]: pgmap v146: 129 pgs: 15 active+undersized+degraded, 15 peering, 2 stale+active+clean, 10 active+undersized, 87 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 88/1389 objects degraded (6.335%) 2023-11-16T13:43:15.205 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.205 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.205 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:12 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.205 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:12 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.205 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:12 smithi151 ceph-mon[155575]: pgmap v147: 129 pgs: 21 active+undersized+degraded, 8 peering, 14 active+undersized, 86 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 124/1389 objects degraded (8.927%) 2023-11-16T13:43:15.205 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:08 smithi167 ceph-osd[150266]: -- 172.21.15.167:0/1927098620 <== mon.1 v2:172.21.15.167:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5618c5c1cb60 con 0x5618c5be2c00 2023-11-16T13:43:15.205 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:08 smithi167 bash[150229]: 6c4e2d28bf30f29d5320000a5b1d78c05926b0dfe04e26a3b0fb87f370362ba1 2023-11-16T13:43:15.205 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:08 smithi167 systemd[1]: Started Ceph osd.6 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:43:15.205 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:43:08.774+0000 7f02d889b540 -1 Falling back to public interface 2023-11-16T13:43:15.206 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:43:15.209 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:09 smithi167 ceph-mon[138945]: pgmap v145: 129 pgs: 15 peering, 10 stale+active+clean, 104 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:43:15.209 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:09 smithi167 ceph-mon[138945]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-11-16T13:43:15.209 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:09 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.209 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:09 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.209 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:09 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:43:15.209 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:10 smithi167 ceph-mon[138945]: Health check failed: Degraded data redundancy: 88/1389 objects degraded (6.335%), 15 pgs degraded (PG_DEGRADED) 2023-11-16T13:43:15.210 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:11 smithi167 ceph-mon[138945]: pgmap v146: 129 pgs: 15 active+undersized+degraded, 15 peering, 2 stale+active+clean, 10 active+undersized, 87 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 88/1389 objects degraded (6.335%) 2023-11-16T13:43:15.210 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.210 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.210 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:12 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.210 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:12 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.210 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:12 smithi167 ceph-mon[138945]: pgmap v147: 129 pgs: 21 active+undersized+degraded, 8 peering, 14 active+undersized, 86 active+clean; 318 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 124/1389 objects degraded (8.927%) 2023-11-16T13:43:15.210 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:13 smithi151 ceph-mon[155575]: from='osd.6 [v2:172.21.15.167:6816/2385104186,v1:172.21.15.167:6817/2385104186]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-11-16T13:43:15.210 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:13 smithi151 ceph-mon[155575]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-11-16T13:43:15.210 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.210 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:43:15.211 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-11-16T13:43:15.211 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: osdmap e100: 8 total, 7 up, 8 in 2023-11-16T13:43:15.211 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='osd.6 [v2:172.21.15.167:6816/2385104186,v1:172.21.15.167:6817/2385104186]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:43:15.211 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:43:15.211 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.211 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:15.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:43:15.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: pgmap v149: 129 pgs: 26 active+undersized+degraded, 17 active+undersized, 86 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 264 B/s wr, 0 op/s; 150/1389 objects degraded (10.799%) 2023-11-16T13:43:15.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:43:15.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:15.240 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:15.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:15.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2023-11-16T13:43:15.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2023-11-16T13:43:15.241 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:14 smithi151 ceph-mon[155575]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2023-11-16T13:43:15.241 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:43:12.982+0000 7f02d889b540 -1 osd.6 97 log_to_monitors true 2023-11-16T13:43:15.241 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:13 smithi167 ceph-mon[138945]: from='osd.6 [v2:172.21.15.167:6816/2385104186,v1:172.21.15.167:6817/2385104186]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-11-16T13:43:15.242 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:13 smithi167 ceph-mon[138945]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-11-16T13:43:15.242 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.242 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:43:15.242 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:43:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:43:13.576+0000 7f02ca67c700 -1 osd.6 97 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-11-16T13:43:15.242 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-11-16T13:43:15.242 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: osdmap e100: 8 total, 7 up, 8 in 2023-11-16T13:43:15.242 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='osd.6 [v2:172.21.15.167:6816/2385104186,v1:172.21.15.167:6817/2385104186]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:43:15.242 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:43:15.243 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.243 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.243 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:15.244 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:43:15.244 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.244 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:15.244 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: pgmap v149: 129 pgs: 26 active+undersized+degraded, 17 active+undersized, 86 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 264 B/s wr, 0 op/s; 150/1389 objects degraded (10.799%) 2023-11-16T13:43:15.244 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:43:15.244 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:15.244 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:15.244 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:15.244 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2023-11-16T13:43:15.244 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2023-11-16T13:43:15.245 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:14 smithi167 ceph-mon[138945]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2023-11-16T13:43:15.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:15 smithi167 ceph-mon[138945]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:43:15.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:15 smithi167 ceph-mon[138945]: osd.6 [v2:172.21.15.167:6816/2385104186,v1:172.21.15.167:6817/2385104186] boot 2023-11-16T13:43:15.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:15 smithi167 ceph-mon[138945]: osdmap e101: 8 total, 8 up, 8 in 2023-11-16T13:43:15.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:43:15.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:15 smithi167 ceph-mon[138945]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-11-16T13:43:15.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:15 smithi167 ceph-mon[138945]: osdmap e102: 8 total, 8 up, 8 in 2023-11-16T13:43:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:15 smithi151 ceph-mon[155575]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:43:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:15 smithi151 ceph-mon[155575]: osd.6 [v2:172.21.15.167:6816/2385104186,v1:172.21.15.167:6817/2385104186] boot 2023-11-16T13:43:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:15 smithi151 ceph-mon[155575]: osdmap e101: 8 total, 8 up, 8 in 2023-11-16T13:43:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-11-16T13:43:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:15 smithi151 ceph-mon[155575]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-11-16T13:43:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:15 smithi151 ceph-mon[155575]: osdmap e102: 8 total, 8 up, 8 in 2023-11-16T13:43:16.850 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:16 smithi167 ceph-mon[138945]: pgmap v152: 129 pgs: 9 peering, 23 active+undersized+degraded, 11 active+undersized, 86 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 133/1389 objects degraded (9.575%) 2023-11-16T13:43:16.851 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:16 smithi167 ceph-mon[138945]: Health check update: Degraded data redundancy: 150/1389 objects degraded (10.799%), 26 pgs degraded (PG_DEGRADED) 2023-11-16T13:43:16.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:16 smithi151 ceph-mon[155575]: pgmap v152: 129 pgs: 9 peering, 23 active+undersized+degraded, 11 active+undersized, 86 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 133/1389 objects degraded (9.575%) 2023-11-16T13:43:16.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:16 smithi151 ceph-mon[155575]: Health check update: Degraded data redundancy: 150/1389 objects degraded (10.799%), 26 pgs degraded (PG_DEGRADED) 2023-11-16T13:43:18.991 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:43:19.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:19 smithi151 ceph-mon[155575]: pgmap v153: 129 pgs: 9 peering, 15 active+undersized+degraded, 7 active+undersized, 98 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s; 89/1389 objects degraded (6.407%) 2023-11-16T13:43:19.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:19 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:19.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:19 smithi167 ceph-mon[138945]: pgmap v153: 129 pgs: 9 peering, 15 active+undersized+degraded, 7 active+undersized, 98 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s; 89/1389 objects degraded (6.407%) 2023-11-16T13:43:19.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:19 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:20.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:20 smithi151 ceph-mon[155575]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 89/1389 objects degraded (6.407%), 15 pgs degraded) 2023-11-16T13:43:20.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:20 smithi151 ceph-mon[155575]: Cluster is now healthy 2023-11-16T13:43:20.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:20 smithi167 ceph-mon[138945]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 89/1389 objects degraded (6.407%), 15 pgs degraded) 2023-11-16T13:43:20.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:20 smithi167 ceph-mon[138945]: Cluster is now healthy 2023-11-16T13:43:21.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:21 smithi151 ceph-mon[155575]: pgmap v154: 129 pgs: 1 peering, 128 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 158 B/s rd, 158 B/s wr, 0 op/s 2023-11-16T13:43:21.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:21 smithi167 ceph-mon[138945]: pgmap v154: 129 pgs: 1 peering, 128 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 158 B/s rd, 158 B/s wr, 0 op/s 2023-11-16T13:43:22.192 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:43:22.815 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:43:22.815 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (11m) 92s ago 12m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:43:22.815 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (3m) 92s ago 12m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:43:22.816 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (3m) 12s ago 11m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:43:22.816 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (11m) 92s ago 12m 36.4M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:43:22.816 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (8m) 92s ago 8m 16.3M - 16.2.5 6933c2a0b7dd acd6cd6834e2 2023-11-16T13:43:22.816 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (8m) 12s ago 8m 42.3M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:43:22.816 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (4m) 92s ago 13m 501M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:43:22.816 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (4m) 12s ago 11m 439M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:43:22.816 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (4m) 92s ago 13m 91.6M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:43:22.816 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (3m) 12s ago 11m 86.4M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:43:22.816 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (6m) 92s ago 6m 85.0M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:43:22.816 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (6m) 12s ago 6m 79.5M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:43:22.816 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (12m) 92s ago 12m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:43:22.816 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (11m) 12s ago 11m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:43:22.816 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (3m) 92s ago 11m 382M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:43:22.817 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (2m) 92s ago 11m 291M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 49b15d0ef2d2 2023-11-16T13:43:22.817 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (2m) 92s ago 10m 216M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 89fcacc46719 2023-11-16T13:43:22.817 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (96s) 92s ago 10m 12.4M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 aa3cf87e766a 2023-11-16T13:43:22.817 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (68s) 12s ago 10m 389M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 cbb1a9fdea87 2023-11-16T13:43:22.817 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (41s) 12s ago 10m 388M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 82902ee5659d 2023-11-16T13:43:22.817 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (14s) 12s ago 10m 14.5M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 6c4e2d28bf30 2023-11-16T13:43:22.817 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (9m) 12s ago 9m 1850M 2712M 16.2.5 6933c2a0b7dd d148ff3820a9 2023-11-16T13:43:22.817 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (4m) 92s ago 12m 64.3M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:43:23.181 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:43:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:43:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:43:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:43:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:43:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:43:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:43:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:43:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1, 2023-11-16T13:43:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 7 2023-11-16T13:43:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:43:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:43:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2023-11-16T13:43:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:43:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:43:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 3, 2023-11-16T13:43:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 11 2023-11-16T13:43:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:43:23.182 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:43:23.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:23 smithi151 ceph-mon[155575]: pgmap v155: 129 pgs: 129 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2023-11-16T13:43:23.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:23 smithi151 ceph-mon[155575]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:23.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:23 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/3692360722' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:23.487 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:43:23.487 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:43:23.487 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:43:23.487 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:43:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:43:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: "crash", 2023-11-16T13:43:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: "mon", 2023-11-16T13:43:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:43:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:43:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "13/23 daemons upgraded", 2023-11-16T13:43:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading osd daemons", 2023-11-16T13:43:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:43:23.488 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:43:23.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:23 smithi167 ceph-mon[138945]: pgmap v155: 129 pgs: 129 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2023-11-16T13:43:23.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:23 smithi167 ceph-mon[138945]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:23.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:23 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/3692360722' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:24.028 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:43:24.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:24 smithi151 ceph-mon[155575]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:24.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:24 smithi151 ceph-mon[155575]: from='client.34302 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:24.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:24 smithi167 ceph-mon[138945]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:24.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:24 smithi167 ceph-mon[138945]: from='client.34302 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:25.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:25 smithi151 ceph-mon[155575]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:25.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:25 smithi151 ceph-mon[155575]: pgmap v156: 129 pgs: 129 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 216 B/s rd, 216 B/s wr, 0 op/s 2023-11-16T13:43:25.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:25 smithi167 ceph-mon[138945]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:25.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:25 smithi167 ceph-mon[138945]: pgmap v156: 129 pgs: 129 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 216 B/s rd, 216 B/s wr, 0 op/s 2023-11-16T13:43:27.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:27 smithi167 ceph-mon[138945]: pgmap v157: 129 pgs: 129 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 195 B/s rd, 195 B/s wr, 0 op/s 2023-11-16T13:43:27.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:27 smithi151 ceph-mon[155575]: pgmap v157: 129 pgs: 129 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 195 B/s rd, 195 B/s wr, 0 op/s 2023-11-16T13:43:29.073 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:43:29.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:29 smithi167 ceph-mon[138945]: pgmap v158: 129 pgs: 129 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:43:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:29 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:29 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:43:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:29 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:29 smithi151 ceph-mon[155575]: pgmap v158: 129 pgs: 129 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:43:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:29 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:29 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:43:29.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:29 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:30.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:30 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2023-11-16T13:43:30.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:30 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2023-11-16T13:43:30.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:30 smithi167 ceph-mon[138945]: Upgrade: osd.7 is safe to restart 2023-11-16T13:43:30.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:30 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2023-11-16T13:43:30.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:30 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2023-11-16T13:43:30.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:30 smithi151 ceph-mon[155575]: Upgrade: osd.7 is safe to restart 2023-11-16T13:43:31.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:31 smithi167 ceph-mon[138945]: pgmap v159: 129 pgs: 129 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:43:31.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:31 smithi167 ceph-mon[138945]: Upgrade: Updating osd.7 2023-11-16T13:43:31.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:31 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:31.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:31 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-11-16T13:43:31.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:31 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:31.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:31 smithi167 ceph-mon[138945]: Deploying daemon osd.7 on smithi167 2023-11-16T13:43:31.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:31 smithi151 ceph-mon[155575]: pgmap v159: 129 pgs: 129 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:43:31.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:31 smithi151 ceph-mon[155575]: Upgrade: Updating osd.7 2023-11-16T13:43:31.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:31 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:31.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:31 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-11-16T13:43:31.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:31 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:31.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:31 smithi151 ceph-mon[155575]: Deploying daemon osd.7 on smithi167 2023-11-16T13:43:32.096 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:31 smithi167 systemd[1]: Stopping Ceph osd.7 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:43:32.097 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.7[123906]: debug 2023-11-16T13:43:32.032+0000 7f4101247700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-11-16T13:43:32.097 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.7[123906]: debug 2023-11-16T13:43:32.032+0000 7f4101247700 -1 osd.7 102 *** Got signal Terminated *** 2023-11-16T13:43:32.097 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:32 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.7[123906]: debug 2023-11-16T13:43:32.032+0000 7f4101247700 -1 osd.7 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:43:32.596 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:32 smithi167 bash[152346]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd.7 2023-11-16T13:43:33.482 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: pgmap v160: 129 pgs: 129 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:43:33.482 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.1 2023-11-16T13:43:33.482 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 failed (root=default,host=smithi167) (connection refused reported by osd.1) 2023-11-16T13:43:33.482 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.0 2023-11-16T13:43:33.482 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.1 2023-11-16T13:43:33.482 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.2 2023-11-16T13:43:33.482 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.3 2023-11-16T13:43:33.483 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.6 2023-11-16T13:43:33.483 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.0 2023-11-16T13:43:33.483 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.2 2023-11-16T13:43:33.483 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.3 2023-11-16T13:43:33.483 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.5 2023-11-16T13:43:33.483 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.6 2023-11-16T13:43:33.483 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.4 2023-11-16T13:43:33.483 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.5 2023-11-16T13:43:33.483 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.4 2023-11-16T13:43:33.483 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.0 2023-11-16T13:43:33.484 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.2 2023-11-16T13:43:33.484 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.2 2023-11-16T13:43:33.484 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.3 2023-11-16T13:43:33.484 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.0 2023-11-16T13:43:33.484 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.1 2023-11-16T13:43:33.484 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.3 2023-11-16T13:43:33.484 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.5 2023-11-16T13:43:33.484 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.0 2023-11-16T13:43:33.484 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.1 2023-11-16T13:43:33.484 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.4 2023-11-16T13:43:33.485 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.5 2023-11-16T13:43:33.485 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.6 2023-11-16T13:43:33.485 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.4 2023-11-16T13:43:33.485 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:33 smithi167 ceph-mon[138945]: osd.7 reported immediately failed by osd.6 2023-11-16T13:43:33.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: pgmap v160: 129 pgs: 129 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 170 B/s wr, 0 op/s 2023-11-16T13:43:33.485 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.1 2023-11-16T13:43:33.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 failed (root=default,host=smithi167) (connection refused reported by osd.1) 2023-11-16T13:43:33.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.0 2023-11-16T13:43:33.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.1 2023-11-16T13:43:33.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.2 2023-11-16T13:43:33.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.3 2023-11-16T13:43:33.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.6 2023-11-16T13:43:33.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.0 2023-11-16T13:43:33.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.2 2023-11-16T13:43:33.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.3 2023-11-16T13:43:33.486 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.5 2023-11-16T13:43:33.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.6 2023-11-16T13:43:33.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.4 2023-11-16T13:43:33.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.5 2023-11-16T13:43:33.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.4 2023-11-16T13:43:33.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.0 2023-11-16T13:43:33.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.2 2023-11-16T13:43:33.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.2 2023-11-16T13:43:33.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.3 2023-11-16T13:43:33.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.0 2023-11-16T13:43:33.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.1 2023-11-16T13:43:33.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.3 2023-11-16T13:43:33.487 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.5 2023-11-16T13:43:33.488 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.0 2023-11-16T13:43:33.488 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.1 2023-11-16T13:43:33.488 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.4 2023-11-16T13:43:33.488 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.5 2023-11-16T13:43:33.488 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.6 2023-11-16T13:43:33.488 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.4 2023-11-16T13:43:33.488 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:33 smithi151 ceph-mon[155575]: osd.7 reported immediately failed by osd.6 2023-11-16T13:43:33.812 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:33 smithi167 systemd[1]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.7.service: Succeeded. 2023-11-16T13:43:33.812 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:33 smithi167 systemd[1]: Stopped Ceph osd.7 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:43:33.812 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:33 smithi167 systemd[1]: Starting Ceph osd.7 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:43:34.111 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:43:34.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:34 smithi151 ceph-mon[155575]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:43:34.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:34 smithi151 ceph-mon[155575]: Health check failed: all OSDs are running quincy or later but require_osd_release < quincy (OSD_UPGRADE_FINISHED) 2023-11-16T13:43:34.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:34 smithi151 ceph-mon[155575]: osdmap e103: 8 total, 7 up, 8 in 2023-11-16T13:43:34.505 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:34 smithi167 ceph-mon[138945]: Health check failed: 1 osds down (OSD_DOWN) 2023-11-16T13:43:34.506 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:34 smithi167 ceph-mon[138945]: Health check failed: all OSDs are running quincy or later but require_osd_release < quincy (OSD_UPGRADE_FINISHED) 2023-11-16T13:43:34.506 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:34 smithi167 ceph-mon[138945]: osdmap e103: 8 total, 7 up, 8 in 2023-11-16T13:43:34.506 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:34 smithi167 bash[152618]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-11-16T13:43:34.506 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:34 smithi167 bash[152618]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2023-11-16T13:43:34.506 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:34 smithi167 bash[152618]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2023-11-16T13:43:34.506 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:34 smithi167 bash[152618]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-11-16T13:43:34.506 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:34 smithi167 bash[152618]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2023-11-16T13:43:34.506 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:34 smithi167 bash[152618]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-11-16T13:43:34.506 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:34 smithi167 bash[152618]: --> ceph-volume raw activate successful for osd ID: 7 2023-11-16T13:43:35.346 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:34 smithi167 ceph-osd[152928]: -- 172.21.15.167:0/2310145918 <== mon.1 v2:172.21.15.167:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5644d7282b60 con 0x5644d7248c00 2023-11-16T13:43:35.347 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:34 smithi167 bash[152890]: 8abcdef4b8d014347c95652ed41e93b81aaa12ae7c4ba7afc74a9463992f6cf8 2023-11-16T13:43:35.347 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:34 smithi167 systemd[1]: Started Ceph osd.7 for 11fdb452-8484-11ee-8dbe-212e2dc638e7. 2023-11-16T13:43:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:35 smithi167 ceph-mon[138945]: pgmap v162: 129 pgs: 8 peering, 15 stale+active+clean, 106 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:43:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:35 smithi167 ceph-mon[138945]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-11-16T13:43:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:35 smithi167 ceph-mon[138945]: osdmap e104: 8 total, 7 up, 8 in 2023-11-16T13:43:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:35 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:35 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:35.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:35 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:43:35.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:35 smithi151 ceph-mon[155575]: pgmap v162: 129 pgs: 8 peering, 15 stale+active+clean, 106 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:43:35.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:35 smithi151 ceph-mon[155575]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-11-16T13:43:35.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:35 smithi151 ceph-mon[155575]: osdmap e104: 8 total, 7 up, 8 in 2023-11-16T13:43:35.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:35 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:35.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:35 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:35.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:35 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:43:35.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:35 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:43:35.489+0000 7fea6b080540 -1 Falling back to public interface 2023-11-16T13:43:37.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:37 smithi167 ceph-mon[138945]: pgmap v164: 129 pgs: 18 active+undersized+degraded, 8 peering, 4 stale+active+clean, 7 active+undersized, 92 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 134/1389 objects degraded (9.647%) 2023-11-16T13:43:37.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:37 smithi167 ceph-mon[138945]: Health check failed: Degraded data redundancy: 134/1389 objects degraded (9.647%), 18 pgs degraded (PG_DEGRADED) 2023-11-16T13:43:37.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:37 smithi151 ceph-mon[155575]: pgmap v164: 129 pgs: 18 active+undersized+degraded, 8 peering, 4 stale+active+clean, 7 active+undersized, 92 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 134/1389 objects degraded (9.647%) 2023-11-16T13:43:37.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:37 smithi151 ceph-mon[155575]: Health check failed: Degraded data redundancy: 134/1389 objects degraded (9.647%), 18 pgs degraded (PG_DEGRADED) 2023-11-16T13:43:38.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:38 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:38.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:38 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:38.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:38 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:38.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:38 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:38.525 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:38 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:38.525 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:38 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:38.525 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:38 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:38.526 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:38 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:39.147 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:43:39.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:39 smithi167 ceph-mon[138945]: pgmap v165: 129 pgs: 32 active+undersized+degraded, 8 peering, 16 active+undersized, 73 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 201/1389 objects degraded (14.471%) 2023-11-16T13:43:39.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:39 smithi151 ceph-mon[155575]: pgmap v165: 129 pgs: 32 active+undersized+degraded, 8 peering, 16 active+undersized, 73 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 201/1389 objects degraded (14.471%) 2023-11-16T13:43:40.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:40 smithi167 ceph-mon[138945]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-11-16T13:43:40.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:40 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:40.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:40 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:40.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:40 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:40.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:40 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:43:40.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:40 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:40.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:40 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:40.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:40 smithi151 ceph-mon[155575]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-11-16T13:43:40.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:40 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:40.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:40 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:40.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:40 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:40.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:40 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:43:40.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:40 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:40.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:40 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:41.096 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:40 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:43:40.691+0000 7fea6b080540 -1 osd.7 102 log_to_monitors true 2023-11-16T13:43:41.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:43:41.460+0000 7fea5e664700 -1 osd.7 102 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-11-16T13:43:41.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: pgmap v166: 129 pgs: 37 active+undersized+degraded, 19 active+undersized, 73 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 219/1389 objects degraded (15.767%) 2023-11-16T13:43:41.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:43:41.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:41.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:41.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:41.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:41.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: Upgrade: Setting container_image for all osd 2023-11-16T13:43:41.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:41.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2023-11-16T13:43:41.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2023-11-16T13:43:41.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2023-11-16T13:43:41.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2023-11-16T13:43:41.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2023-11-16T13:43:41.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2023-11-16T13:43:41.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2023-11-16T13:43:41.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2023-11-16T13:43:41.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2023-11-16T13:43:41.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2023-11-16T13:43:41.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2023-11-16T13:43:41.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2023-11-16T13:43:41.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2023-11-16T13:43:41.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2023-11-16T13:43:41.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2023-11-16T13:43:41.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2023-11-16T13:43:41.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: Upgrade: Setting require_osd_release to 17 quincy 2023-11-16T13:43:41.599 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd require-osd-release", "release": "quincy"}]: dispatch 2023-11-16T13:43:41.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='osd.7 [v2:172.21.15.167:6824/2955522196,v1:172.21.15.167:6825/2955522196]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-11-16T13:43:41.600 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:41 smithi167 ceph-mon[138945]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-11-16T13:43:41.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: pgmap v166: 129 pgs: 37 active+undersized+degraded, 19 active+undersized, 73 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 219/1389 objects degraded (15.767%) 2023-11-16T13:43:41.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:43:41.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:41.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:41.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:41.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:41.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: Upgrade: Setting container_image for all osd 2023-11-16T13:43:41.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:41.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2023-11-16T13:43:41.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2023-11-16T13:43:41.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2023-11-16T13:43:41.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2023-11-16T13:43:41.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2023-11-16T13:43:41.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2023-11-16T13:43:41.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2023-11-16T13:43:41.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2023-11-16T13:43:41.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2023-11-16T13:43:41.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2023-11-16T13:43:41.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2023-11-16T13:43:41.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2023-11-16T13:43:41.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2023-11-16T13:43:41.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2023-11-16T13:43:41.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2023-11-16T13:43:41.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2023-11-16T13:43:41.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: Upgrade: Setting require_osd_release to 17 quincy 2023-11-16T13:43:41.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd require-osd-release", "release": "quincy"}]: dispatch 2023-11-16T13:43:41.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='osd.7 [v2:172.21.15.167:6824/2955522196,v1:172.21.15.167:6825/2955522196]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-11-16T13:43:41.740 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:41 smithi151 ceph-mon[155575]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-11-16T13:43:42.439 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:42 smithi151 ceph-mon[155575]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running quincy or later but require_osd_release < quincy) 2023-11-16T13:43:42.439 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd require-osd-release", "release": "quincy"}]': finished 2023-11-16T13:43:42.439 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:42 smithi151 ceph-mon[155575]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-11-16T13:43:42.439 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:42 smithi151 ceph-mon[155575]: osdmap e105: 8 total, 7 up, 8 in 2023-11-16T13:43:42.439 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:42 smithi151 ceph-mon[155575]: from='osd.7 [v2:172.21.15.167:6824/2955522196,v1:172.21.15.167:6825/2955522196]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:43:42.439 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:42 smithi151 ceph-mon[155575]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:43:42.440 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi151.mdkcfw"]}]: dispatch 2023-11-16T13:43:42.440 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:42 smithi151 ceph-mon[155575]: Upgrade: It appears safe to stop mds.foofs.smithi151.mdkcfw 2023-11-16T13:43:42.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:42 smithi167 ceph-mon[138945]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running quincy or later but require_osd_release < quincy) 2023-11-16T13:43:42.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "osd require-osd-release", "release": "quincy"}]': finished 2023-11-16T13:43:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:42 smithi167 ceph-mon[138945]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-11-16T13:43:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:42 smithi167 ceph-mon[138945]: osdmap e105: 8 total, 7 up, 8 in 2023-11-16T13:43:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:42 smithi167 ceph-mon[138945]: from='osd.7 [v2:172.21.15.167:6824/2955522196,v1:172.21.15.167:6825/2955522196]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:43:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:42 smithi167 ceph-mon[138945]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi167", "root=default"]}]: dispatch 2023-11-16T13:43:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi151.mdkcfw"]}]: dispatch 2023-11-16T13:43:42.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:42 smithi167 ceph-mon[138945]: Upgrade: It appears safe to stop mds.foofs.smithi151.mdkcfw 2023-11-16T13:43:43.477 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:43 smithi151 ceph-mon[155575]: pgmap v168: 129 pgs: 37 active+undersized+degraded, 19 active+undersized, 73 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 219/1389 objects degraded (15.767%) 2023-11-16T13:43:43.477 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:43 smithi151 ceph-mon[155575]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:43:43.477 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:43 smithi151 ceph-mon[155575]: osd.7 [v2:172.21.15.167:6824/2955522196,v1:172.21.15.167:6825/2955522196] boot 2023-11-16T13:43:43.478 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:43 smithi151 ceph-mon[155575]: osdmap e106: 8 total, 8 up, 8 in 2023-11-16T13:43:43.478 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:43:43.478 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:43 smithi151 ceph-mon[155575]: Upgrade: Updating mds.foofs.smithi151.mdkcfw 2023-11-16T13:43:43.478 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:43.478 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi151.mdkcfw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-11-16T13:43:43.478 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:43.478 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:43.478 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:43:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:43 smithi167 ceph-mon[138945]: pgmap v168: 129 pgs: 37 active+undersized+degraded, 19 active+undersized, 73 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 219/1389 objects degraded (15.767%) 2023-11-16T13:43:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:43 smithi167 ceph-mon[138945]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-11-16T13:43:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:43 smithi167 ceph-mon[138945]: osd.7 [v2:172.21.15.167:6824/2955522196,v1:172.21.15.167:6825/2955522196] boot 2023-11-16T13:43:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:43 smithi167 ceph-mon[138945]: osdmap e106: 8 total, 8 up, 8 in 2023-11-16T13:43:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-11-16T13:43:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:43 smithi167 ceph-mon[138945]: Upgrade: Updating mds.foofs.smithi151.mdkcfw 2023-11-16T13:43:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi151.mdkcfw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-11-16T13:43:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:43:44.187 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:43:44.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:44 smithi151 ceph-mon[155575]: Deploying daemon mds.foofs.smithi151.mdkcfw on smithi151 2023-11-16T13:43:44.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:44 smithi151 ceph-mon[155575]: osdmap e107: 8 total, 8 up, 8 in 2023-11-16T13:43:44.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:44 smithi151 ceph-mon[155575]: daemon mds.foofs.smithi151.mdkcfw compat changed unexpectedly 2023-11-16T13:43:44.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:44 smithi167 ceph-mon[138945]: Deploying daemon mds.foofs.smithi151.mdkcfw on smithi151 2023-11-16T13:43:44.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:44 smithi167 ceph-mon[138945]: osdmap e107: 8 total, 8 up, 8 in 2023-11-16T13:43:44.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:44 smithi167 ceph-mon[138945]: daemon mds.foofs.smithi151.mdkcfw compat changed unexpectedly 2023-11-16T13:43:45.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:45 smithi151 ceph-mon[155575]: pgmap v171: 129 pgs: 3 peering, 32 active+undersized+degraded, 15 active+undersized, 79 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s; 203/1389 objects degraded (14.615%) 2023-11-16T13:43:45.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:45 smithi151 ceph-mon[155575]: Health check update: Degraded data redundancy: 203/1389 objects degraded (14.615%), 32 pgs degraded (PG_DEGRADED) 2023-11-16T13:43:45.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:45 smithi151 ceph-mon[155575]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2023-11-16T13:43:45.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:45 smithi151 ceph-mon[155575]: osdmap e108: 8 total, 8 up, 8 in 2023-11-16T13:43:45.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:45 smithi151 ceph-mon[155575]: fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 2023-11-16T13:43:45.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:45 smithi167 ceph-mon[138945]: pgmap v171: 129 pgs: 3 peering, 32 active+undersized+degraded, 15 active+undersized, 79 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s; 203/1389 objects degraded (14.615%) 2023-11-16T13:43:45.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:45 smithi167 ceph-mon[138945]: Health check update: Degraded data redundancy: 203/1389 objects degraded (14.615%), 32 pgs degraded (PG_DEGRADED) 2023-11-16T13:43:45.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:45 smithi167 ceph-mon[138945]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2023-11-16T13:43:45.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:45 smithi167 ceph-mon[138945]: osdmap e108: 8 total, 8 up, 8 in 2023-11-16T13:43:45.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:45 smithi167 ceph-mon[138945]: fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 2023-11-16T13:43:47.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:47 smithi151 ceph-mon[155575]: pgmap v173: 129 pgs: 3 peering, 23 active+undersized+degraded, 10 active+undersized, 93 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 223 B/s rd, 223 B/s wr, 0 op/s; 141/1389 objects degraded (10.151%) 2023-11-16T13:43:47.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:47.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:47.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:47 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:43:47.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:47 smithi167 ceph-mon[138945]: pgmap v173: 129 pgs: 3 peering, 23 active+undersized+degraded, 10 active+undersized, 93 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 223 B/s rd, 223 B/s wr, 0 op/s; 141/1389 objects degraded (10.151%) 2023-11-16T13:43:47.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:47.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:47.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:47 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:43:48.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:48 smithi151 ceph-mon[155575]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2023-11-16T13:43:48.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:48 smithi151 ceph-mon[155575]: mds.? [v2:172.21.15.151:6834/3146709773,v1:172.21.15.151:6835/3146709773] up:boot 2023-11-16T13:43:48.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:48 smithi151 ceph-mon[155575]: fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 1 up:standby 2023-11-16T13:43:48.700 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:48 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi151.mdkcfw"}]: dispatch 2023-11-16T13:43:48.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:48 smithi167 ceph-mon[138945]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2023-11-16T13:43:48.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:48 smithi167 ceph-mon[138945]: mds.? [v2:172.21.15.151:6834/3146709773,v1:172.21.15.151:6835/3146709773] up:boot 2023-11-16T13:43:48.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:48 smithi167 ceph-mon[138945]: fsmap foofs:1 {0=foofs.smithi167.irznoa=up:active} 1 up:standby 2023-11-16T13:43:48.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:48 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi151.mdkcfw"}]: dispatch 2023-11-16T13:43:49.226 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:43:49.697 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:49 smithi151 ceph-mon[155575]: pgmap v174: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2023-11-16T13:43:49.697 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:49 smithi151 ceph-mon[155575]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 141/1389 objects degraded (10.151%), 23 pgs degraded) 2023-11-16T13:43:49.697 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:49 smithi151 ceph-mon[155575]: Cluster is now healthy 2023-11-16T13:43:49.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:49 smithi167 ceph-mon[138945]: pgmap v174: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2023-11-16T13:43:49.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:49 smithi167 ceph-mon[138945]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 141/1389 objects degraded (10.151%), 23 pgs degraded) 2023-11-16T13:43:49.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:49 smithi167 ceph-mon[138945]: Cluster is now healthy 2023-11-16T13:43:51.689 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:51 smithi151 ceph-mon[155575]: pgmap v175: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 135 B/s rd, 270 B/s wr, 0 op/s 2023-11-16T13:43:51.689 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:51 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:51.689 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:51 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:51.689 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:51 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:51.689 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:51 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:51.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:51 smithi167 ceph-mon[138945]: pgmap v175: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 135 B/s rd, 270 B/s wr, 0 op/s 2023-11-16T13:43:51.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:51 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:51.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:51 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:51.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:51 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:51.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:51 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:53 smithi151 ceph-mon[155575]: pgmap v176: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 238 B/s rd, 119 B/s wr, 0 op/s 2023-11-16T13:43:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:43:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:53.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:53 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:53.823 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:43:53.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:53 smithi167 ceph-mon[138945]: pgmap v176: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 238 B/s rd, 119 B/s wr, 0 op/s 2023-11-16T13:43:53.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:53.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:53.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:53.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:53.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:43:53.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:53.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:53 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:54.259 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:43:54.439 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:43:54.440 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (11m) 4s ago 13m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:43:54.440 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (3m) 4s ago 13m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:43:54.440 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (3m) 17s ago 12m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:43:54.440 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (11m) 4s ago 12m 36.7M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:43:54.440 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (7s) 4s ago 9m 15.5M - 17.2.7-68-gae6ce92c ea201341e8f3 811376ceeaa9 2023-11-16T13:43:54.440 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (9m) 17s ago 9m 42.3M - 16.2.5 6933c2a0b7dd 2f54addca368 2023-11-16T13:43:54.440 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (5m) 4s ago 14m 510M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:43:54.440 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (4m) 17s ago 12m 440M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:43:54.440 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (4m) 4s ago 14m 100M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:43:54.440 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (4m) 17s ago 12m 75.3M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:43:54.440 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (7m) 4s ago 7m 87.4M - 3.5 6933c2a0b7dd d18ce4a85488 2023-11-16T13:43:54.441 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (7m) 17s ago 7m 79.6M - 3.5 6933c2a0b7dd 1221457bd118 2023-11-16T13:43:54.441 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (12m) 4s ago 12m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:43:54.441 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (12m) 17s ago 12m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:43:54.441 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (3m) 4s ago 11m 415M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:43:54.441 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (3m) 4s ago 11m 322M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 49b15d0ef2d2 2023-11-16T13:43:54.441 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (2m) 4s ago 11m 240M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 89fcacc46719 2023-11-16T13:43:54.441 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (2m) 4s ago 11m 347M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 aa3cf87e766a 2023-11-16T13:43:54.441 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (100s) 17s ago 11m 398M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 cbb1a9fdea87 2023-11-16T13:43:54.441 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (73s) 17s ago 10m 394M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 82902ee5659d 2023-11-16T13:43:54.441 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (46s) 17s ago 10m 307M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 6c4e2d28bf30 2023-11-16T13:43:54.441 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (19s) 17s ago 10m 11.7M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 8abcdef4b8d0 2023-11-16T13:43:54.441 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (5m) 4s ago 12m 70.5M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:43:54.489 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:54 smithi167 ceph-mon[138945]: Detected new or changed devices on smithi151 2023-11-16T13:43:54.490 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:43:54.490 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:54.490 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:54.490 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:54.490 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:54.490 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:54 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi167.irznoa"]}]: dispatch 2023-11-16T13:43:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:54 smithi151 ceph-mon[155575]: Detected new or changed devices on smithi151 2023-11-16T13:43:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:43:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:54.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:54 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi167.irznoa"]}]: dispatch 2023-11-16T13:43:54.807 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:43:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:43:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:43:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:43:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:43:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:43:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:43:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:43:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 8 2023-11-16T13:43:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:43:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:43:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1, 2023-11-16T13:43:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 1 2023-11-16T13:43:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:43:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:43:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1, 2023-11-16T13:43:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 13 2023-11-16T13:43:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:43:54.809 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:43:55.117 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:43:55.117 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:43:55.117 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:43:55.117 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:43:55.117 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:43:55.117 INFO:teuthology.orchestra.run.smithi151.stdout: "crash", 2023-11-16T13:43:55.118 INFO:teuthology.orchestra.run.smithi151.stdout: "mon", 2023-11-16T13:43:55.118 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr", 2023-11-16T13:43:55.118 INFO:teuthology.orchestra.run.smithi151.stdout: "osd" 2023-11-16T13:43:55.118 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:43:55.118 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "15/23 daemons upgraded", 2023-11-16T13:43:55.118 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading mds daemons", 2023-11-16T13:43:55.118 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:43:55.118 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:43:55.517 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:55 smithi167 ceph-mon[138945]: Upgrade: It appears safe to stop mds.foofs.smithi167.irznoa 2023-11-16T13:43:55.518 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:55 smithi167 ceph-mon[138945]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:55.518 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:55 smithi167 ceph-mon[138945]: pgmap v177: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:43:55.518 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:55 smithi167 ceph-mon[138945]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:55.518 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:55 smithi167 ceph-mon[138945]: from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:55.518 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:55 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:55.518 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:55 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi167.irznoa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-11-16T13:43:55.518 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:55 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:55.518 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:55 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/3618382858' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:55 smithi151 ceph-mon[155575]: Upgrade: It appears safe to stop mds.foofs.smithi167.irznoa 2023-11-16T13:43:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:55 smithi151 ceph-mon[155575]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:55 smithi151 ceph-mon[155575]: pgmap v177: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:43:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:55 smithi151 ceph-mon[155575]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:55 smithi151 ceph-mon[155575]: from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:55 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:55 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi167.irznoa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-11-16T13:43:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:55 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:43:55.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:55 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/3618382858' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:43:55.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:55 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon-smithi151[155552]: 2023-11-16T13:43:55.638+0000 7fa803fd5700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-11-16T13:43:55.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:55 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:43:55.662+0000 7fea46e35700 -1 bdev(0x5644d8048400 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 8 2023-11-16T13:43:55.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:43:55 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:43:55.664+0000 7fea49e3b700 -1 bdev(0x5644d8048400 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 13 2023-11-16T13:43:56.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:56 smithi151 ceph-mon[155575]: Upgrade: Updating mds.foofs.smithi167.irznoa 2023-11-16T13:43:56.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:56 smithi151 ceph-mon[155575]: Deploying daemon mds.foofs.smithi167.irznoa on smithi167 2023-11-16T13:43:56.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:56 smithi151 ceph-mon[155575]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:56.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:56 smithi151 ceph-mon[155575]: daemon mds.foofs.smithi167.irznoa compat changed unexpectedly 2023-11-16T13:43:56.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:56 smithi151 ceph-mon[155575]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2023-11-16T13:43:56.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:56 smithi151 ceph-mon[155575]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-11-16T13:43:56.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:56 smithi151 ceph-mon[155575]: osdmap e109: 8 total, 8 up, 8 in 2023-11-16T13:43:56.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:56 smithi151 ceph-mon[155575]: Standby daemon mds.foofs.smithi151.mdkcfw assigned to filesystem foofs as rank 0 2023-11-16T13:43:56.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:56 smithi151 ceph-mon[155575]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2023-11-16T13:43:56.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:56 smithi151 ceph-mon[155575]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-11-16T13:43:56.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:56 smithi151 ceph-mon[155575]: fsmap foofs:0/1 1 up:standby, 1 failed 2023-11-16T13:43:56.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:56 smithi151 ceph-mon[155575]: fsmap foofs:1/1 {0=foofs.smithi151.mdkcfw=up:replay} 2023-11-16T13:43:56.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:56 smithi167 ceph-mon[138945]: Upgrade: Updating mds.foofs.smithi167.irznoa 2023-11-16T13:43:56.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:56 smithi167 ceph-mon[138945]: Deploying daemon mds.foofs.smithi167.irznoa on smithi167 2023-11-16T13:43:56.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:56 smithi167 ceph-mon[138945]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:43:56.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:56 smithi167 ceph-mon[138945]: daemon mds.foofs.smithi167.irznoa compat changed unexpectedly 2023-11-16T13:43:56.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:56 smithi167 ceph-mon[138945]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2023-11-16T13:43:56.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:56 smithi167 ceph-mon[138945]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-11-16T13:43:56.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:56 smithi167 ceph-mon[138945]: osdmap e109: 8 total, 8 up, 8 in 2023-11-16T13:43:56.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:56 smithi167 ceph-mon[138945]: Standby daemon mds.foofs.smithi151.mdkcfw assigned to filesystem foofs as rank 0 2023-11-16T13:43:56.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:56 smithi167 ceph-mon[138945]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2023-11-16T13:43:56.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:56 smithi167 ceph-mon[138945]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-11-16T13:43:56.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:56 smithi167 ceph-mon[138945]: fsmap foofs:0/1 1 up:standby, 1 failed 2023-11-16T13:43:56.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:56 smithi167 ceph-mon[138945]: fsmap foofs:1/1 {0=foofs.smithi151.mdkcfw=up:replay} 2023-11-16T13:43:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:57 smithi151 ceph-mon[155575]: pgmap v179: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:43:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:57 smithi151 ceph-mon[155575]: mds.? [v2:172.21.15.151:6834/3146709773,v1:172.21.15.151:6835/3146709773] up:reconnect 2023-11-16T13:43:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:57 smithi151 ceph-mon[155575]: fsmap foofs:1/1 {0=foofs.smithi151.mdkcfw=up:reconnect} 2023-11-16T13:43:57.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:57 smithi167 ceph-mon[138945]: pgmap v179: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2023-11-16T13:43:57.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:57 smithi167 ceph-mon[138945]: mds.? [v2:172.21.15.151:6834/3146709773,v1:172.21.15.151:6835/3146709773] up:reconnect 2023-11-16T13:43:57.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:57 smithi167 ceph-mon[138945]: fsmap foofs:1/1 {0=foofs.smithi151.mdkcfw=up:reconnect} 2023-11-16T13:43:59.296 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:43:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:59.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:43:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:59 smithi167 ceph-mon[138945]: pgmap v180: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 204 B/s wr, 4 op/s 2023-11-16T13:43:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:59 smithi167 ceph-mon[138945]: mds.? [v2:172.21.15.151:6834/3146709773,v1:172.21.15.151:6835/3146709773] up:rejoin 2023-11-16T13:43:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:59 smithi167 ceph-mon[138945]: fsmap foofs:1/1 {0=foofs.smithi151.mdkcfw=up:rejoin} 2023-11-16T13:43:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:59 smithi167 ceph-mon[138945]: daemon mds.foofs.smithi151.mdkcfw is now active in filesystem foofs as rank 0 2023-11-16T13:43:59.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:43:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:43:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:43:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:43:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:59 smithi151 ceph-mon[155575]: pgmap v180: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 204 B/s wr, 4 op/s 2023-11-16T13:43:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:59 smithi151 ceph-mon[155575]: mds.? [v2:172.21.15.151:6834/3146709773,v1:172.21.15.151:6835/3146709773] up:rejoin 2023-11-16T13:43:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:59 smithi151 ceph-mon[155575]: fsmap foofs:1/1 {0=foofs.smithi151.mdkcfw=up:rejoin} 2023-11-16T13:43:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:59 smithi151 ceph-mon[155575]: daemon mds.foofs.smithi151.mdkcfw is now active in filesystem foofs as rank 0 2023-11-16T13:43:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:43:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:44:00.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:00 smithi167 ceph-mon[138945]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2023-11-16T13:44:00.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:00 smithi167 ceph-mon[138945]: mds.? [v2:172.21.15.151:6834/3146709773,v1:172.21.15.151:6835/3146709773] up:active 2023-11-16T13:44:00.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:00 smithi167 ceph-mon[138945]: fsmap foofs:1 {0=foofs.smithi151.mdkcfw=up:active} 2023-11-16T13:44:00.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:00 smithi151 ceph-mon[155575]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2023-11-16T13:44:00.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:00 smithi151 ceph-mon[155575]: mds.? [v2:172.21.15.151:6834/3146709773,v1:172.21.15.151:6835/3146709773] up:active 2023-11-16T13:44:00.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:00 smithi151 ceph-mon[155575]: fsmap foofs:1 {0=foofs.smithi151.mdkcfw=up:active} 2023-11-16T13:44:01.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:01 smithi167 ceph-mon[138945]: pgmap v181: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 204 B/s wr, 7 op/s 2023-11-16T13:44:01.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:01 smithi151 ceph-mon[155575]: pgmap v181: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 204 B/s wr, 7 op/s 2023-11-16T13:44:03.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:03 smithi151 ceph-mon[155575]: pgmap v182: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 204 B/s wr, 8 op/s 2023-11-16T13:44:03.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:03 smithi167 ceph-mon[138945]: pgmap v182: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 204 B/s wr, 8 op/s 2023-11-16T13:44:04.337 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:44:05.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:05 smithi151 ceph-mon[155575]: pgmap v183: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 307 B/s wr, 10 op/s 2023-11-16T13:44:05.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:05 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:05.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:05 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:05.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:05 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:44:05.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:05 smithi167 ceph-mon[138945]: pgmap v183: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 307 B/s wr, 10 op/s 2023-11-16T13:44:05.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:05 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:05.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:05 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:05.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:05 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:44:06.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:06 smithi151 ceph-mon[155575]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2023-11-16T13:44:06.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:06 smithi151 ceph-mon[155575]: Cluster is now healthy 2023-11-16T13:44:06.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:06 smithi151 ceph-mon[155575]: mds.? [v2:172.21.15.167:6832/4228475920,v1:172.21.15.167:6833/4228475920] up:boot 2023-11-16T13:44:06.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:06 smithi151 ceph-mon[155575]: fsmap foofs:1 {0=foofs.smithi151.mdkcfw=up:active} 1 up:standby 2023-11-16T13:44:06.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:06 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:44:06.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:06 smithi167 ceph-mon[138945]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2023-11-16T13:44:06.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:06 smithi167 ceph-mon[138945]: Cluster is now healthy 2023-11-16T13:44:06.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:06 smithi167 ceph-mon[138945]: mds.? [v2:172.21.15.167:6832/4228475920,v1:172.21.15.167:6833/4228475920] up:boot 2023-11-16T13:44:06.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:06 smithi167 ceph-mon[138945]: fsmap foofs:1 {0=foofs.smithi151.mdkcfw=up:active} 1 up:standby 2023-11-16T13:44:06.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:06 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:44:07.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:07 smithi167 ceph-mon[138945]: pgmap v184: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 295 B/s wr, 10 op/s 2023-11-16T13:44:07.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:07 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:07.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:07 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:07 smithi151 ceph-mon[155575]: pgmap v184: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 295 B/s wr, 10 op/s 2023-11-16T13:44:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:07 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:07 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:08.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:08 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:08.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:08 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:08.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:08 smithi151 ceph-mon[155575]: pgmap v185: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 6.7 KiB/s wr, 12 op/s 2023-11-16T13:44:09.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:08 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:09.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:08 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:09.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:08 smithi167 ceph-mon[138945]: pgmap v185: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 6.7 KiB/s wr, 12 op/s 2023-11-16T13:44:09.383 INFO:teuthology.orchestra.run.smithi151.stderr:mount.nfs: mounting smithi151:/fake failed, reason given by server: No such file or directory 2023-11-16T13:44:11.160 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: Detected new or changed devices on smithi167 2023-11-16T13:44:11.160 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.160 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.160 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:44:11.160 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:44:11.160 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.161 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.161 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: pgmap v186: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 6.6 KiB/s wr, 8 op/s 2023-11-16T13:44:11.161 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:44:11.161 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.162 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.162 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.162 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.162 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.163 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: Upgrade: Setting container_image for all mds 2023-11-16T13:44:11.163 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.163 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi151.mdkcfw"}]: dispatch 2023-11-16T13:44:11.163 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi151.mdkcfw"}]': finished 2023-11-16T13:44:11.164 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:44:11.164 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi167.irznoa"}]': finished 2023-11-16T13:44:11.164 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.164 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: Upgrade: Setting container_image for all rgw 2023-11-16T13:44:11.164 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.164 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.164 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: Upgrade: Setting container_image for all rbd-mirror 2023-11-16T13:44:11.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: Upgrade: Setting container_image for all iscsi 2023-11-16T13:44:11.165 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: Detected new or changed devices on smithi167 2023-11-16T13:44:11.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:44:11.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:44:11.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: pgmap v186: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 6.6 KiB/s wr, 8 op/s 2023-11-16T13:44:11.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:44:11.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: Upgrade: Setting container_image for all mds 2023-11-16T13:44:11.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi151.mdkcfw"}]: dispatch 2023-11-16T13:44:11.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi151.mdkcfw"}]': finished 2023-11-16T13:44:11.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi167.irznoa"}]: dispatch 2023-11-16T13:44:11.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi167.irznoa"}]': finished 2023-11-16T13:44:11.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: Upgrade: Setting container_image for all rgw 2023-11-16T13:44:11.350 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.350 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.350 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: Upgrade: Setting container_image for all rbd-mirror 2023-11-16T13:44:11.350 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:11.350 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.350 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:11.350 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: Upgrade: Setting container_image for all iscsi 2023-11-16T13:44:11.351 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:12.389 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: Upgrade: Updating nfs.foo.0.1.smithi151.qfkjic (1/2) 2023-11-16T13:44:12.389 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:12.389 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: Creating key for client.nfs.foo.0.1.smithi151.qfkjic 2023-11-16T13:44:12.389 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi151.qfkjic", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2023-11-16T13:44:12.390 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: Ensuring nfs.foo.0 is in the ganesha grace table 2023-11-16T13:44:12.390 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2023-11-16T13:44:12.390 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2023-11-16T13:44:12.390 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:44:12.390 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2023-11-16T13:44:12.390 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2023-11-16T13:44:12.390 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: Rados config object exists: conf-nfs.foo 2023-11-16T13:44:12.390 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: Creating key for client.nfs.foo.0.1.smithi151.qfkjic-rgw 2023-11-16T13:44:12.390 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi151.qfkjic-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2023-11-16T13:44:12.390 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: Bind address in nfs.foo.0.1.smithi151.qfkjic's ganesha conf is defaulting to empty 2023-11-16T13:44:12.390 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:44:12.390 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:12 smithi151 ceph-mon[155575]: Deploying daemon nfs.foo.0.1.smithi151.qfkjic on smithi151 2023-11-16T13:44:12.571 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: Upgrade: Updating nfs.foo.0.1.smithi151.qfkjic (1/2) 2023-11-16T13:44:12.571 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:12.571 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: Creating key for client.nfs.foo.0.1.smithi151.qfkjic 2023-11-16T13:44:12.571 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi151.qfkjic", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2023-11-16T13:44:12.571 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: Ensuring nfs.foo.0 is in the ganesha grace table 2023-11-16T13:44:12.571 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2023-11-16T13:44:12.572 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2023-11-16T13:44:12.572 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:44:12.572 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2023-11-16T13:44:12.572 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2023-11-16T13:44:12.572 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: Rados config object exists: conf-nfs.foo 2023-11-16T13:44:12.572 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: Creating key for client.nfs.foo.0.1.smithi151.qfkjic-rgw 2023-11-16T13:44:12.572 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi151.qfkjic-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2023-11-16T13:44:12.572 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: Bind address in nfs.foo.0.1.smithi151.qfkjic's ganesha conf is defaulting to empty 2023-11-16T13:44:12.572 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:44:12.572 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:12 smithi167 ceph-mon[138945]: Deploying daemon nfs.foo.0.1.smithi151.qfkjic on smithi151 2023-11-16T13:44:13.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:13 smithi167 ceph-mon[138945]: pgmap v187: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 345 KiB/s rd, 6.6 KiB/s wr, 6 op/s 2023-11-16T13:44:13.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:13.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:44:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:13 smithi151 ceph-mon[155575]: pgmap v187: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 345 KiB/s rd, 6.6 KiB/s wr, 6 op/s 2023-11-16T13:44:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:44:14.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:14.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:14 smithi167 ceph-mon[138945]: pgmap v188: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 6.7 KiB/s wr, 5 op/s 2023-11-16T13:44:14.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:14.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:14.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:14 smithi151 ceph-mon[155575]: pgmap v188: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 6.7 KiB/s wr, 5 op/s 2023-11-16T13:44:15.725 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: Upgrade: Updating nfs.foo.1.1.smithi167.rpoanb (2/2) 2023-11-16T13:44:15.725 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:15.725 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: Creating key for client.nfs.foo.1.1.smithi167.rpoanb 2023-11-16T13:44:15.725 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi167.rpoanb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2023-11-16T13:44:15.725 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: Ensuring nfs.foo.1 is in the ganesha grace table 2023-11-16T13:44:15.725 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2023-11-16T13:44:15.726 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2023-11-16T13:44:15.726 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:44:15.726 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2023-11-16T13:44:15.726 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2023-11-16T13:44:15.726 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: Rados config object exists: conf-nfs.foo 2023-11-16T13:44:15.726 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: Creating key for client.nfs.foo.1.1.smithi167.rpoanb-rgw 2023-11-16T13:44:15.726 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi167.rpoanb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2023-11-16T13:44:15.726 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: Bind address in nfs.foo.1.1.smithi167.rpoanb's ganesha conf is defaulting to empty 2023-11-16T13:44:15.727 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:44:15.727 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:15 smithi167 ceph-mon[138945]: Deploying daemon nfs.foo.1.1.smithi167.rpoanb on smithi167 2023-11-16T13:44:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: Upgrade: Updating nfs.foo.1.1.smithi167.rpoanb (2/2) 2023-11-16T13:44:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: Creating key for client.nfs.foo.1.1.smithi167.rpoanb 2023-11-16T13:44:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi167.rpoanb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2023-11-16T13:44:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: Ensuring nfs.foo.1 is in the ganesha grace table 2023-11-16T13:44:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2023-11-16T13:44:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2023-11-16T13:44:15.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:44:15.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2023-11-16T13:44:15.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2023-11-16T13:44:15.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: Rados config object exists: conf-nfs.foo 2023-11-16T13:44:15.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: Creating key for client.nfs.foo.1.1.smithi167.rpoanb-rgw 2023-11-16T13:44:15.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi167.rpoanb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2023-11-16T13:44:15.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: Bind address in nfs.foo.1.1.smithi167.rpoanb's ganesha conf is defaulting to empty 2023-11-16T13:44:15.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:44:15.985 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:15 smithi151 ceph-mon[155575]: Deploying daemon nfs.foo.1.1.smithi167.rpoanb on smithi167 2023-11-16T13:44:17.447 DEBUG:teuthology.orchestra.run.smithi151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2023-11-16T13:44:17.475 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:17 smithi151 ceph-mon[155575]: pgmap v189: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 6.6 KiB/s wr, 3 op/s 2023-11-16T13:44:17.498 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:17 smithi167 ceph-mon[138945]: pgmap v189: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 6.6 KiB/s wr, 3 op/s 2023-11-16T13:44:17.511 INFO:teuthology.orchestra.run.smithi151.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2023-11-16T13:44:17.511 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:44:17.511 INFO:teuthology.orchestra.run.smithi151.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2023-11-16T13:44:17.512 INFO:teuthology.orchestra.run.smithi151.stdout:3 of 5 processes prepared for launch 0 sec 2023-11-16T13:44:17.612 INFO:teuthology.orchestra.run.smithi151.stdout:5 of 5 processes prepared for launch 0 sec 2023-11-16T13:44:17.612 INFO:teuthology.orchestra.run.smithi151.stdout:releasing clients 2023-11-16T13:44:18.612 INFO:teuthology.orchestra.run.smithi151.stdout: 0 1 0.00 MB/sec warmup 1 sec latency 1000.174 ms 2023-11-16T13:44:18.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:18.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:18.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:44:18.702 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:18 smithi151 ceph-mon[155575]: pgmap v190: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 8.8 KiB/s wr, 9 op/s 2023-11-16T13:44:19.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:19.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:19.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:44:19.098 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:18 smithi167 ceph-mon[138945]: pgmap v190: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 8.8 KiB/s wr, 9 op/s 2023-11-16T13:44:19.613 INFO:teuthology.orchestra.run.smithi151.stdout: 0 1 0.00 MB/sec warmup 2 sec latency 2000.233 ms 2023-11-16T13:44:20.612 INFO:teuthology.orchestra.run.smithi151.stdout: 0 1 0.00 MB/sec warmup 3 sec latency 3000.261 ms 2023-11-16T13:44:20.794 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:20.794 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:20.794 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:20 smithi151 ceph-mon[155575]: pgmap v191: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 2.6 KiB/s wr, 7 op/s 2023-11-16T13:44:20.795 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:20.795 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:20 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:20.799 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:20.799 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:20.800 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:20 smithi167 ceph-mon[138945]: pgmap v191: 129 pgs: 129 active+clean; 318 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 2.6 KiB/s wr, 7 op/s 2023-11-16T13:44:20.800 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:20.800 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:20 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:21.612 INFO:teuthology.orchestra.run.smithi151.stdout: 4 90 4.91 MB/sec warmup 4 sec latency 4000.277 ms 2023-11-16T13:44:22.101 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:21 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:22.101 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:21 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:22.286 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:21 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:22.286 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:21 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:23.118 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:22 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:23.118 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:22 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:23.118 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:22 smithi151 ceph-mon[155575]: pgmap v192: 129 pgs: 129 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 574 KiB/s wr, 22 op/s 2023-11-16T13:44:23.118 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:22 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:23.118 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:22 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:22 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:22 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:23.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:22 smithi167 ceph-mon[138945]: pgmap v192: 129 pgs: 129 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 574 KiB/s wr, 22 op/s 2023-11-16T13:44:23.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:22 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:23.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:22 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:23.613 INFO:teuthology.orchestra.run.smithi151.stdout: 5 475 27.93 MB/sec execute 1 sec latency 113.837 ms 2023-11-16T13:44:24.613 INFO:teuthology.orchestra.run.smithi151.stdout: 5 720 25.95 MB/sec execute 2 sec latency 109.379 ms 2023-11-16T13:44:25.170 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:25 smithi151 ceph-mon[155575]: pgmap v193: 129 pgs: 129 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 31 KiB/s rd, 1.4 MiB/s wr, 32 op/s 2023-11-16T13:44:25.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:25 smithi167 ceph-mon[138945]: pgmap v193: 129 pgs: 129 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 31 KiB/s rd, 1.4 MiB/s wr, 32 op/s 2023-11-16T13:44:25.620 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:44:25.621 INFO:teuthology.orchestra.run.smithi151.stdout: 5 1027 19.91 MB/sec execute 3 sec latency 101.590 ms 2023-11-16T13:44:26.592 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:44:26.592 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (12m) 5s ago 13m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:44:26.592 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (4m) 5s ago 13m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:44:26.593 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (4m) 7s ago 12m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:44:26.593 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (12m) 5s ago 13m 36.9M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:44:26.593 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (39s) 5s ago 10m 30.9M - 17.2.7-68-gae6ce92c ea201341e8f3 811376ceeaa9 2023-11-16T13:44:26.593 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (21s) 7s ago 9m 12.9M - 17.2.7-68-gae6ce92c ea201341e8f3 36ac20a06016 2023-11-16T13:44:26.593 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (5m) 5s ago 14m 513M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:44:26.593 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (5m) 7s ago 12m 440M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:44:26.593 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (5m) 5s ago 14m 106M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:44:26.593 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (4m) 7s ago 12m 80.4M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:44:26.593 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (13s) 5s ago 7m 57.7M - 4.0 ea201341e8f3 4d8e7f0525d7 2023-11-16T13:44:26.593 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (9s) 7s ago 7m 20.3M - 4.0 ea201341e8f3 6b233b631706 2023-11-16T13:44:26.593 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (13m) 5s ago 13m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:44:26.594 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (12m) 7s ago 12m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:44:26.594 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (4m) 5s ago 12m 421M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:44:26.594 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (3m) 5s ago 12m 329M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 49b15d0ef2d2 2023-11-16T13:44:26.594 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (3m) 5s ago 11m 246M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 89fcacc46719 2023-11-16T13:44:26.594 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (2m) 5s ago 11m 357M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 aa3cf87e766a 2023-11-16T13:44:26.594 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (2m) 7s ago 11m 408M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 cbb1a9fdea87 2023-11-16T13:44:26.594 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (105s) 7s ago 11m 405M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 82902ee5659d 2023-11-16T13:44:26.595 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (78s) 7s ago 11m 317M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 6c4e2d28bf30 2023-11-16T13:44:26.595 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (51s) 7s ago 10m 480M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 8abcdef4b8d0 2023-11-16T13:44:26.595 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (5m) 5s ago 13m 75.0M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:44:26.630 INFO:teuthology.orchestra.run.smithi151.stdout: 5 1384 17.92 MB/sec execute 4 sec latency 93.849 ms 2023-11-16T13:44:27.200 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:44:27.200 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:44:27.200 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:44:27.200 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:44:27.200 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:44:27.200 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:44:27.200 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:44:27.200 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:44:27.200 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 8 2023-11-16T13:44:27.201 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:44:27.201 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:44:27.201 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:44:27.201 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:44:27.201 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:44:27.201 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 14 2023-11-16T13:44:27.201 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:44:27.201 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:44:27.468 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='client.44291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:44:27.468 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: pgmap v194: 129 pgs: 129 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 37 KiB/s rd, 4.8 MiB/s wr, 159 op/s 2023-11-16T13:44:27.468 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:44:27.468 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:27.468 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:27.469 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:44:27.469 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:44:27.469 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:27.469 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:27.469 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:44:27.469 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.469 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.470 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.470 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.470 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.470 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.470 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.470 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.470 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.471 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.471 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:27.471 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.1.smithi151.qfkjic"}]: dispatch 2023-11-16T13:44:27.471 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.1.smithi151.qfkjic"}]': finished 2023-11-16T13:44:27.471 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.1.smithi167.rpoanb"}]: dispatch 2023-11-16T13:44:27.471 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.1.smithi167.rpoanb"}]': finished 2023-11-16T13:44:27.471 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:27 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/1537758611' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.630 INFO:teuthology.orchestra.run.smithi151.stdout: 5 cleanup 5 sec 2023-11-16T13:44:27.686 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:44:27.686 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:44:27.686 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:44:27.686 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:44:27.686 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:44:27.686 INFO:teuthology.orchestra.run.smithi151.stdout: "crash", 2023-11-16T13:44:27.687 INFO:teuthology.orchestra.run.smithi151.stdout: "mon", 2023-11-16T13:44:27.687 INFO:teuthology.orchestra.run.smithi151.stdout: "nfs", 2023-11-16T13:44:27.687 INFO:teuthology.orchestra.run.smithi151.stdout: "mds", 2023-11-16T13:44:27.687 INFO:teuthology.orchestra.run.smithi151.stdout: "osd", 2023-11-16T13:44:27.687 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:44:27.687 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:44:27.687 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "18/23 daemons upgraded", 2023-11-16T13:44:27.688 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading node-exporter daemons", 2023-11-16T13:44:27.688 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:44:27.688 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:44:27.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='client.44291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:44:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: pgmap v194: 129 pgs: 129 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 37 KiB/s rd, 4.8 MiB/s wr, 159 op/s 2023-11-16T13:44:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:44:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:44:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:44:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:44:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:44:27.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.1.smithi151.qfkjic"}]: dispatch 2023-11-16T13:44:27.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.1.smithi151.qfkjic"}]': finished 2023-11-16T13:44:27.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.1.smithi167.rpoanb"}]: dispatch 2023-11-16T13:44:27.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.1.smithi167.rpoanb"}]': finished 2023-11-16T13:44:27.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:27 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/1537758611' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:27.855 INFO:teuthology.orchestra.run.smithi151.stdout:[1580] unlink './clients/client3/~dmtmp/PARADOX/.nfs0000010000003edc00000016' failed - Device or resource busy 2023-11-16T13:44:27.856 INFO:teuthology.orchestra.run.smithi151.stdout:[1580] unlink './clients/client3/~dmtmp/PARADOX/.nfs0000010000003ef00000001f' failed - Device or resource busy 2023-11-16T13:44:27.857 INFO:teuthology.orchestra.run.smithi151.stdout:[1580] unlink './clients/client3/~dmtmp/PARADOX/.nfs0000010000003ee40000001e' failed - Device or resource busy 2023-11-16T13:44:27.857 INFO:teuthology.orchestra.run.smithi151.stdout:[1580] unlink './clients/client3/~dmtmp/PARADOX/.nfs0000010000003eee00000022' failed - Device or resource busy 2023-11-16T13:44:27.879 INFO:teuthology.orchestra.run.smithi151.stdout:[1580] unlink './clients/client3/~dmtmp/PARADOX/.nfs0000010000003ee900000023' failed - Device or resource busy 2023-11-16T13:44:27.880 INFO:teuthology.orchestra.run.smithi151.stdout:[1580] unlink './clients/client3/~dmtmp/PARADOX/.nfs0000010000003ecc0000001a' failed - Device or resource busy 2023-11-16T13:44:27.880 INFO:teuthology.orchestra.run.smithi151.stdout:[1580] unlink './clients/client3/~dmtmp/PARADOX/.nfs0000010000003ee700000020' failed - Device or resource busy 2023-11-16T13:44:27.895 INFO:teuthology.orchestra.run.smithi151.stdout:[1580] unlink './clients/client3/~dmtmp/PARADOX/.nfs0000010000003eec00000018' failed - Device or resource busy 2023-11-16T13:44:28.500 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:28 smithi151 ceph-mon[155575]: from='client.44295 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:44:28.500 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:28 smithi151 ceph-mon[155575]: Upgrade: Setting container_image for all nfs 2023-11-16T13:44:28.501 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:44:28.633 INFO:teuthology.orchestra.run.smithi151.stdout: 2 cleanup 6 sec 2023-11-16T13:44:28.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:28 smithi167 ceph-mon[138945]: from='client.44295 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:44:28.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:28 smithi167 ceph-mon[138945]: Upgrade: Setting container_image for all nfs 2023-11-16T13:44:28.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:44:29.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 7 sec 2023-11-16T13:44:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:29 smithi151 ceph-mon[155575]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:44:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:29 smithi151 ceph-mon[155575]: pgmap v195: 129 pgs: 129 active+clean; 322 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 39 KiB/s rd, 9.2 MiB/s wr, 275 op/s 2023-11-16T13:44:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:29 smithi151 ceph-mon[155575]: Upgrade: Updating node-exporter.smithi151 (1/2) 2023-11-16T13:44:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:29 smithi151 ceph-mon[155575]: Deploying daemon node-exporter.smithi151 on smithi151 2023-11-16T13:44:29.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:29 smithi167 ceph-mon[138945]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:44:29.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:29 smithi167 ceph-mon[138945]: pgmap v195: 129 pgs: 129 active+clean; 322 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 39 KiB/s rd, 9.2 MiB/s wr, 275 op/s 2023-11-16T13:44:29.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:29 smithi167 ceph-mon[138945]: Upgrade: Updating node-exporter.smithi151 (1/2) 2023-11-16T13:44:29.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:29 smithi167 ceph-mon[138945]: Deploying daemon node-exporter.smithi151 on smithi151 2023-11-16T13:44:30.633 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 8 sec 2023-11-16T13:44:31.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 9 sec 2023-11-16T13:44:31.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:31 smithi151 ceph-mon[155575]: pgmap v196: 129 pgs: 129 active+clean; 317 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 34 KiB/s rd, 9.8 MiB/s wr, 310 op/s 2023-11-16T13:44:31.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:31 smithi167 ceph-mon[138945]: pgmap v196: 129 pgs: 129 active+clean; 317 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 34 KiB/s rd, 9.8 MiB/s wr, 310 op/s 2023-11-16T13:44:32.633 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 10 sec 2023-11-16T13:44:33.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 11 sec 2023-11-16T13:44:33.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:33 smithi151 ceph-mon[155575]: pgmap v197: 129 pgs: 129 active+clean; 267 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 31 KiB/s rd, 9.9 MiB/s wr, 330 op/s 2023-11-16T13:44:33.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:33 smithi167 ceph-mon[138945]: pgmap v197: 129 pgs: 129 active+clean; 267 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 31 KiB/s rd, 9.9 MiB/s wr, 330 op/s 2023-11-16T13:44:34.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 12 sec 2023-11-16T13:44:35.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 13 sec 2023-11-16T13:44:35.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:35 smithi151 ceph-mon[155575]: pgmap v198: 129 pgs: 129 active+clean; 234 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 9.5 MiB/s wr, 330 op/s 2023-11-16T13:44:35.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:35 smithi167 ceph-mon[138945]: pgmap v198: 129 pgs: 129 active+clean; 234 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 9.5 MiB/s wr, 330 op/s 2023-11-16T13:44:36.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 14 sec 2023-11-16T13:44:37.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 15 sec 2023-11-16T13:44:37.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:37 smithi151 ceph-mon[155575]: pgmap v199: 129 pgs: 129 active+clean; 225 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.7 MiB/s wr, 323 op/s 2023-11-16T13:44:37.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:37 smithi167 ceph-mon[138945]: pgmap v199: 129 pgs: 129 active+clean; 225 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.7 MiB/s wr, 323 op/s 2023-11-16T13:44:38.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 16 sec 2023-11-16T13:44:39.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 17 sec 2023-11-16T13:44:39.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:39 smithi151 ceph-mon[155575]: pgmap v200: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 5.2 MiB/s wr, 198 op/s 2023-11-16T13:44:39.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:39 smithi167 ceph-mon[138945]: pgmap v200: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 5.2 MiB/s wr, 198 op/s 2023-11-16T13:44:40.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 18 sec 2023-11-16T13:44:41.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 19 sec 2023-11-16T13:44:41.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:41 smithi151 ceph-mon[155575]: pgmap v201: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 806 KiB/s wr, 81 op/s 2023-11-16T13:44:41.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:41 smithi167 ceph-mon[138945]: pgmap v201: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 806 KiB/s wr, 81 op/s 2023-11-16T13:44:42.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 20 sec 2023-11-16T13:44:43.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 21 sec 2023-11-16T13:44:43.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:43 smithi151 ceph-mon[155575]: pgmap v202: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 255 KiB/s wr, 41 op/s 2023-11-16T13:44:43.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:44:43.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:43 smithi167 ceph-mon[138945]: pgmap v202: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 255 KiB/s wr, 41 op/s 2023-11-16T13:44:43.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:44:44.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 22 sec 2023-11-16T13:44:45.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 23 sec 2023-11-16T13:44:45.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:45 smithi151 ceph-mon[155575]: pgmap v203: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 153 KiB/s wr, 20 op/s 2023-11-16T13:44:45.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:45 smithi167 ceph-mon[138945]: pgmap v203: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 153 KiB/s wr, 20 op/s 2023-11-16T13:44:46.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 24 sec 2023-11-16T13:44:47.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 25 sec 2023-11-16T13:44:47.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:47 smithi151 ceph-mon[155575]: pgmap v204: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 7.2 KiB/s wr, 5 op/s 2023-11-16T13:44:47.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:47 smithi167 ceph-mon[138945]: pgmap v204: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 7.2 KiB/s wr, 5 op/s 2023-11-16T13:44:48.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 26 sec 2023-11-16T13:44:49.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 27 sec 2023-11-16T13:44:49.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:49 smithi151 ceph-mon[155575]: pgmap v205: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 7.2 KiB/s wr, 2 op/s 2023-11-16T13:44:49.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:49 smithi167 ceph-mon[138945]: pgmap v205: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 7.2 KiB/s wr, 2 op/s 2023-11-16T13:44:50.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 28 sec 2023-11-16T13:44:51.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 29 sec 2023-11-16T13:44:51.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:51 smithi151 ceph-mon[155575]: pgmap v206: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:44:51.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:51 smithi167 ceph-mon[138945]: pgmap v206: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:44:52.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 30 sec 2023-11-16T13:44:53.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 31 sec 2023-11-16T13:44:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:53 smithi151 ceph-mon[155575]: pgmap v207: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:44:53.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:53 smithi167 ceph-mon[138945]: pgmap v207: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:44:54.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 32 sec 2023-11-16T13:44:55.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 33 sec 2023-11-16T13:44:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:55 smithi151 ceph-mon[155575]: pgmap v208: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:44:55.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:55 smithi167 ceph-mon[138945]: pgmap v208: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:44:56.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 34 sec 2023-11-16T13:44:57.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 35 sec 2023-11-16T13:44:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:57 smithi151 ceph-mon[155575]: pgmap v209: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:44:57.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:57 smithi167 ceph-mon[138945]: pgmap v209: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:44:58.025 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:44:58.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 36 sec 2023-11-16T13:44:58.651 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:44:58.651 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (13m) 37s ago 14m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (4m) 37s ago 14m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (4m) 39s ago 13m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (12m) 37s ago 13m 36.9M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (71s) 37s ago 10m 30.9M - 17.2.7-68-gae6ce92c ea201341e8f3 811376ceeaa9 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (53s) 39s ago 10m 12.9M - 17.2.7-68-gae6ce92c ea201341e8f3 36ac20a06016 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (6m) 37s ago 15m 513M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (5m) 39s ago 13m 440M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (5m) 37s ago 15m 106M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (5m) 39s ago 13m 80.4M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (45s) 37s ago 8m 57.7M - 4.0 ea201341e8f3 4d8e7f0525d7 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (41s) 39s ago 8m 20.3M - 4.0 ea201341e8f3 6b233b631706 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (13m) 37s ago 13m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (13m) 39s ago 13m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:44:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (4m) 37s ago 13m 421M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:44:58.653 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (4m) 37s ago 12m 329M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 49b15d0ef2d2 2023-11-16T13:44:58.653 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (3m) 37s ago 12m 246M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 89fcacc46719 2023-11-16T13:44:58.653 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (3m) 37s ago 12m 357M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 aa3cf87e766a 2023-11-16T13:44:58.653 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (2m) 39s ago 12m 408M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 cbb1a9fdea87 2023-11-16T13:44:58.653 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (2m) 39s ago 11m 405M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 82902ee5659d 2023-11-16T13:44:58.653 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (110s) 39s ago 11m 317M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 6c4e2d28bf30 2023-11-16T13:44:58.653 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (83s) 39s ago 11m 480M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 8abcdef4b8d0 2023-11-16T13:44:58.653 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (6m) 37s ago 13m 75.0M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:44:58.699 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:58 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:44:58.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:58 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:44:59.023 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:44:59.024 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:44:59.024 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:44:59.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:44:59.024 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:44:59.024 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:44:59.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:44:59.024 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:44:59.024 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 8 2023-11-16T13:44:59.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:44:59.024 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:44:59.024 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:44:59.025 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:44:59.025 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:44:59.025 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 14 2023-11-16T13:44:59.025 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:44:59.025 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:44:59.338 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:44:59.339 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:44:59.339 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:44:59.339 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:44:59.339 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:44:59.339 INFO:teuthology.orchestra.run.smithi151.stdout: "crash", 2023-11-16T13:44:59.339 INFO:teuthology.orchestra.run.smithi151.stdout: "mon", 2023-11-16T13:44:59.339 INFO:teuthology.orchestra.run.smithi151.stdout: "nfs", 2023-11-16T13:44:59.339 INFO:teuthology.orchestra.run.smithi151.stdout: "mds", 2023-11-16T13:44:59.339 INFO:teuthology.orchestra.run.smithi151.stdout: "osd", 2023-11-16T13:44:59.339 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:44:59.339 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:44:59.339 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "18/23 daemons upgraded", 2023-11-16T13:44:59.339 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading node-exporter daemons", 2023-11-16T13:44:59.340 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:44:59.340 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:44:59.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 37 sec 2023-11-16T13:44:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:59 smithi151 ceph-mon[155575]: from='client.34400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:44:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:59 smithi151 ceph-mon[155575]: pgmap v210: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:44:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:59 smithi151 ceph-mon[155575]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:44:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:44:59 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/1230125572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:44:59.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:59 smithi167 ceph-mon[138945]: from='client.34400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:44:59.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:59 smithi167 ceph-mon[138945]: pgmap v210: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2023-11-16T13:44:59.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:59 smithi167 ceph-mon[138945]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:44:59.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:44:59 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/1230125572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:45:00.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 38 sec 2023-11-16T13:45:00.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:00 smithi151 ceph-mon[155575]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:45:00.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:00 smithi151 ceph-mon[155575]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:45:00.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:00 smithi167 ceph-mon[138945]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:45:00.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:00 smithi167 ceph-mon[138945]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:45:01.634 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 39 sec 2023-11-16T13:45:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:01 smithi151 ceph-mon[155575]: pgmap v211: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:01.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:01 smithi167 ceph-mon[138945]: pgmap v211: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:02.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 40 sec 2023-11-16T13:45:03.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 41 sec 2023-11-16T13:45:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:03 smithi151 ceph-mon[155575]: pgmap v212: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:45:03.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:03 smithi167 ceph-mon[138945]: pgmap v212: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:45:04.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 42 sec 2023-11-16T13:45:05.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 43 sec 2023-11-16T13:45:05.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:05 smithi151 ceph-mon[155575]: pgmap v213: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:05.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:05 smithi167 ceph-mon[138945]: pgmap v213: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:06.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 44 sec 2023-11-16T13:45:07.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 45 sec 2023-11-16T13:45:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:07 smithi151 ceph-mon[155575]: pgmap v214: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:07.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:07 smithi167 ceph-mon[138945]: pgmap v214: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:08.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 46 sec 2023-11-16T13:45:09.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 47 sec 2023-11-16T13:45:09.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:09 smithi151 ceph-mon[155575]: pgmap v215: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:09.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:09 smithi167 ceph-mon[138945]: pgmap v215: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:10.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 48 sec 2023-11-16T13:45:11.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 49 sec 2023-11-16T13:45:11.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:11 smithi151 ceph-mon[155575]: pgmap v216: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:11.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:11 smithi167 ceph-mon[138945]: pgmap v216: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:12.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 50 sec 2023-11-16T13:45:13.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 51 sec 2023-11-16T13:45:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:13 smithi151 ceph-mon[155575]: pgmap v217: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:45:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:45:13.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:13 smithi167 ceph-mon[138945]: pgmap v217: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:45:13.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:45:14.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 52 sec 2023-11-16T13:45:15.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 53 sec 2023-11-16T13:45:15.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:15 smithi151 ceph-mon[155575]: pgmap v218: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:15.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:15 smithi167 ceph-mon[138945]: pgmap v218: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:16.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 54 sec 2023-11-16T13:45:17.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 55 sec 2023-11-16T13:45:17.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:17 smithi151 ceph-mon[155575]: pgmap v219: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:17.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:17 smithi167 ceph-mon[138945]: pgmap v219: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:18.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 56 sec 2023-11-16T13:45:19.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 57 sec 2023-11-16T13:45:19.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:19 smithi151 ceph-mon[155575]: pgmap v220: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:19.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:19 smithi167 ceph-mon[138945]: pgmap v220: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:20.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 58 sec 2023-11-16T13:45:21.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 59 sec 2023-11-16T13:45:21.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:21 smithi151 ceph-mon[155575]: pgmap v221: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:29.395 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 60 sec 2023-11-16T13:45:29.395 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 61 sec 2023-11-16T13:45:29.396 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:23 smithi151 ceph-mon[155575]: pgmap v222: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:45:29.396 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:21 smithi167 ceph-mon[138945]: pgmap v221: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:29.420 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:23 smithi167 ceph-mon[138945]: pgmap v222: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:45:29.458 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:24 smithi167 ceph-mon[138945]: pgmap v223: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:29.458 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:27 smithi167 ceph-mon[138945]: pgmap v224: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:29.458 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:45:29.459 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 62 sec 2023-11-16T13:45:29.459 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:24 smithi151 ceph-mon[155575]: pgmap v223: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:29.460 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 63 sec 2023-11-16T13:45:29.460 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 64 sec 2023-11-16T13:45:29.461 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:27 smithi151 ceph-mon[155575]: pgmap v224: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:29.461 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 65 sec 2023-11-16T13:45:29.461 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 66 sec 2023-11-16T13:45:29.462 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:45:29.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:29 smithi167 ceph-mon[138945]: pgmap v225: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:29.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 67 sec 2023-11-16T13:45:29.668 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:45:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:29 smithi151 ceph-mon[155575]: pgmap v225: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:30.466 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:45:30.466 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (13m) 69s ago 14m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:45:30.466 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (5m) 69s ago 14m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:45:30.466 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (5m) 70s ago 14m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:45:30.466 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (13m) 69s ago 14m 36.9M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:45:30.466 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (103s) 69s ago 11m 30.9M - 17.2.7-68-gae6ce92c ea201341e8f3 811376ceeaa9 2023-11-16T13:45:30.466 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (85s) 70s ago 11m 12.9M - 17.2.7-68-gae6ce92c ea201341e8f3 36ac20a06016 2023-11-16T13:45:30.466 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (6m) 69s ago 15m 513M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:45:30.466 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (6m) 70s ago 14m 440M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:45:30.466 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (6m) 69s ago 15m 106M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (5m) 70s ago 13m 80.4M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (77s) 69s ago 8m 57.7M - 4.0 ea201341e8f3 4d8e7f0525d7 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (73s) 70s ago 8m 20.3M - 4.0 ea201341e8f3 6b233b631706 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (14m) 69s ago 14m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (13m) 70s ago 13m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (5m) 69s ago 13m 421M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (4m) 69s ago 13m 329M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 49b15d0ef2d2 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (4m) 69s ago 13m 246M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 89fcacc46719 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (3m) 69s ago 12m 357M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 aa3cf87e766a 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (3m) 70s ago 12m 408M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 cbb1a9fdea87 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (2m) 70s ago 12m 405M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 82902ee5659d 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (2m) 70s ago 12m 317M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 6c4e2d28bf30 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (115s) 70s ago 11m 480M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 8abcdef4b8d0 2023-11-16T13:45:30.467 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (6m) 69s ago 14m 75.0M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:45:30.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 68 sec 2023-11-16T13:45:30.841 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:45:30.841 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:45:30.841 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:45:30.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:45:30.842 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:45:30.842 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:45:30.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:45:30.842 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:45:30.842 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 8 2023-11-16T13:45:30.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:45:30.842 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:45:30.842 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:45:30.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:45:30.842 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:45:30.842 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 14 2023-11-16T13:45:30.843 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:45:30.843 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:45:31.156 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:45:31.156 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:45:31.156 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:45:31.156 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:45:31.156 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:45:31.156 INFO:teuthology.orchestra.run.smithi151.stdout: "crash", 2023-11-16T13:45:31.157 INFO:teuthology.orchestra.run.smithi151.stdout: "mon", 2023-11-16T13:45:31.157 INFO:teuthology.orchestra.run.smithi151.stdout: "nfs", 2023-11-16T13:45:31.157 INFO:teuthology.orchestra.run.smithi151.stdout: "mds", 2023-11-16T13:45:31.157 INFO:teuthology.orchestra.run.smithi151.stdout: "osd", 2023-11-16T13:45:31.157 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:45:31.157 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:45:31.157 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "18/23 daemons upgraded", 2023-11-16T13:45:31.157 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading node-exporter daemons", 2023-11-16T13:45:31.158 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:45:31.158 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:45:31.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:31 smithi167 ceph-mon[138945]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:45:31.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:31 smithi167 ceph-mon[138945]: pgmap v226: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:31.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:31 smithi167 ceph-mon[138945]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:45:31.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:31 smithi167 ceph-mon[138945]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:45:31.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:31 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/2847815117' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:45:31.635 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 69 sec 2023-11-16T13:45:31.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:31 smithi151 ceph-mon[155575]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:45:31.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:31 smithi151 ceph-mon[155575]: pgmap v226: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:31.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:31 smithi151 ceph-mon[155575]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:45:31.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:31 smithi151 ceph-mon[155575]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:45:31.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:31 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/2847815117' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:45:32.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:32 smithi167 ceph-mon[138945]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:45:32.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 70 sec 2023-11-16T13:45:32.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:32 smithi151 ceph-mon[155575]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:45:33.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:33 smithi167 ceph-mon[138945]: pgmap v227: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:45:33.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 71 sec 2023-11-16T13:45:33.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:33 smithi151 ceph-mon[155575]: pgmap v227: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:45:34.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 72 sec 2023-11-16T13:45:35.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:35 smithi167 ceph-mon[138945]: pgmap v228: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:35.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 73 sec 2023-11-16T13:45:35.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:35 smithi151 ceph-mon[155575]: pgmap v228: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:36.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 74 sec 2023-11-16T13:45:37.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:37 smithi167 ceph-mon[138945]: pgmap v229: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:37.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 75 sec 2023-11-16T13:45:37.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:37 smithi151 ceph-mon[155575]: pgmap v229: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:38.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 76 sec 2023-11-16T13:45:39.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:39 smithi167 ceph-mon[138945]: pgmap v230: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:39.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 77 sec 2023-11-16T13:45:39.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:39 smithi151 ceph-mon[155575]: pgmap v230: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:40.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 78 sec 2023-11-16T13:45:41.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:41 smithi167 ceph-mon[138945]: pgmap v231: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:41.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 79 sec 2023-11-16T13:45:41.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:41 smithi151 ceph-mon[155575]: pgmap v231: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:42.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 80 sec 2023-11-16T13:45:43.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:43 smithi167 ceph-mon[138945]: pgmap v232: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:45:43.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:45:43.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 81 sec 2023-11-16T13:45:43.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:43 smithi151 ceph-mon[155575]: pgmap v232: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:45:43.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:45:44.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 82 sec 2023-11-16T13:45:45.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:45 smithi167 ceph-mon[138945]: pgmap v233: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:45.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 83 sec 2023-11-16T13:45:45.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:45 smithi151 ceph-mon[155575]: pgmap v233: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:46.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 84 sec 2023-11-16T13:45:47.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:47 smithi167 ceph-mon[138945]: pgmap v234: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:47.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 85 sec 2023-11-16T13:45:47.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:47 smithi151 ceph-mon[155575]: pgmap v234: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:48.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 86 sec 2023-11-16T13:45:49.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:49 smithi167 ceph-mon[138945]: pgmap v235: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:49.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 87 sec 2023-11-16T13:45:49.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:49 smithi151 ceph-mon[155575]: pgmap v235: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:50.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 88 sec 2023-11-16T13:45:51.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:51 smithi167 ceph-mon[138945]: pgmap v236: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:51.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 89 sec 2023-11-16T13:45:51.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:51 smithi151 ceph-mon[155575]: pgmap v236: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:52.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 90 sec 2023-11-16T13:45:53.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:53 smithi167 ceph-mon[138945]: pgmap v237: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:45:53.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 91 sec 2023-11-16T13:45:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:53 smithi151 ceph-mon[155575]: pgmap v237: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:45:54.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 92 sec 2023-11-16T13:45:55.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:55 smithi167 ceph-mon[138945]: pgmap v238: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:55.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 93 sec 2023-11-16T13:45:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:55 smithi151 ceph-mon[155575]: pgmap v238: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:56.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 94 sec 2023-11-16T13:45:57.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:57 smithi167 ceph-mon[138945]: pgmap v239: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:57.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 95 sec 2023-11-16T13:45:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:57 smithi151 ceph-mon[155575]: pgmap v239: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:58.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:58 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:45:58.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 96 sec 2023-11-16T13:45:58.699 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:58 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:45:59.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:45:59 smithi167 ceph-mon[138945]: pgmap v240: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:45:59.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 97 sec 2023-11-16T13:45:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:45:59 smithi151 ceph-mon[155575]: pgmap v240: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:00.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 98 sec 2023-11-16T13:46:01.489 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:46:01.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:01 smithi167 ceph-mon[138945]: pgmap v241: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:01.636 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 99 sec 2023-11-16T13:46:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:01 smithi151 ceph-mon[155575]: pgmap v241: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:02.116 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:46:02.116 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (14m) 101s ago 15m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:46:02.116 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (5m) 101s ago 15m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:46:02.116 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (5m) 102s ago 14m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:46:02.116 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (13m) 101s ago 14m 36.9M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:46:02.116 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (2m) 101s ago 11m 30.9M - 17.2.7-68-gae6ce92c ea201341e8f3 811376ceeaa9 2023-11-16T13:46:02.116 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (117s) 102s ago 11m 12.9M - 17.2.7-68-gae6ce92c ea201341e8f3 36ac20a06016 2023-11-16T13:46:02.116 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (7m) 101s ago 16m 513M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:46:02.117 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (6m) 102s ago 14m 440M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:46:02.117 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (6m) 101s ago 16m 106M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:46:02.117 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (6m) 102s ago 14m 80.4M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:46:02.117 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (108s) 101s ago 9m 57.7M - 4.0 ea201341e8f3 4d8e7f0525d7 2023-11-16T13:46:02.117 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (104s) 102s ago 9m 20.3M - 4.0 ea201341e8f3 6b233b631706 2023-11-16T13:46:02.117 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (14m) 101s ago 14m 18.2M - 0.18.1 e5a616e4b9cf 075ac332a356 2023-11-16T13:46:02.117 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (14m) 102s ago 14m 18.3M - 0.18.1 e5a616e4b9cf f190b560a248 2023-11-16T13:46:02.117 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (5m) 101s ago 14m 421M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:46:02.117 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (5m) 101s ago 13m 329M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 49b15d0ef2d2 2023-11-16T13:46:02.117 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (4m) 101s ago 13m 246M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 89fcacc46719 2023-11-16T13:46:02.117 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (4m) 101s ago 13m 357M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 aa3cf87e766a 2023-11-16T13:46:02.117 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (3m) 102s ago 13m 408M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 cbb1a9fdea87 2023-11-16T13:46:02.118 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (3m) 102s ago 12m 405M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 82902ee5659d 2023-11-16T13:46:02.118 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (2m) 102s ago 12m 317M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 6c4e2d28bf30 2023-11-16T13:46:02.118 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (2m) 102s ago 12m 480M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 8abcdef4b8d0 2023-11-16T13:46:02.118 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (7m) 101s ago 14m 75.0M - 2.18.1 de242295e225 1f9732869cba 2023-11-16T13:46:02.523 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:46:02.523 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:46:02.523 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:46:02.523 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:46:02.523 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:46:02.524 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:46:02.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:46:02.524 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:46:02.524 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 8 2023-11-16T13:46:02.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:46:02.524 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:46:02.524 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:46:02.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:46:02.524 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:46:02.524 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 14 2023-11-16T13:46:02.524 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:46:02.524 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:46:02.585 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:02 smithi151 ceph-mon[155575]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:02.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:02 smithi167 ceph-mon[138945]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:02.642 INFO:teuthology.orchestra.run.smithi151.stdout: 1 cleanup 100 sec 2023-11-16T13:46:02.694 INFO:teuthology.orchestra.run.smithi151.stdout: 0 cleanup 100 sec 2023-11-16T13:46:02.694 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:46:02.694 INFO:teuthology.orchestra.run.smithi151.stdout: Operation Count AvgLat MaxLat 2023-11-16T13:46:02.694 INFO:teuthology.orchestra.run.smithi151.stdout: ---------------------------------------- 2023-11-16T13:46:02.694 INFO:teuthology.orchestra.run.smithi151.stdout: NTCreateX 1129 9.135 113.834 2023-11-16T13:46:02.694 INFO:teuthology.orchestra.run.smithi151.stdout: Close 955 3.946 43.248 2023-11-16T13:46:02.694 INFO:teuthology.orchestra.run.smithi151.stdout: Rename 52 12.054 82.032 2023-11-16T13:46:02.694 INFO:teuthology.orchestra.run.smithi151.stdout: Unlink 109 5.806 68.498 2023-11-16T13:46:02.694 INFO:teuthology.orchestra.run.smithi151.stdout: Qpathinfo 755 3.134 63.873 2023-11-16T13:46:02.694 INFO:teuthology.orchestra.run.smithi151.stdout: Qfileinfo 370 7.926 36.982 2023-11-16T13:46:02.695 INFO:teuthology.orchestra.run.smithi151.stdout: Qfsinfo 165 2.552 28.347 2023-11-16T13:46:02.695 INFO:teuthology.orchestra.run.smithi151.stdout: Sfileinfo 170 6.799 40.129 2023-11-16T13:46:02.695 INFO:teuthology.orchestra.run.smithi151.stdout: Find 303 6.694 94.642 2023-11-16T13:46:02.695 INFO:teuthology.orchestra.run.smithi151.stdout: WriteX 1566 0.022 0.062 2023-11-16T13:46:02.695 INFO:teuthology.orchestra.run.smithi151.stdout: ReadX 1303 0.017 2.103 2023-11-16T13:46:02.695 INFO:teuthology.orchestra.run.smithi151.stdout: Flush 131 7.541 23.375 2023-11-16T13:46:02.695 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:46:02.695 INFO:teuthology.orchestra.run.smithi151.stdout:Throughput 17.9152 MB/sec 5 clients 5 procs max_latency=113.837 ms 2023-11-16T13:46:02.696 DEBUG:teuthology.parallel:result is None 2023-11-16T13:46:02.845 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:46:02.845 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:46:02.845 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:46:02.845 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:46:02.845 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:46:02.845 INFO:teuthology.orchestra.run.smithi151.stdout: "crash", 2023-11-16T13:46:02.845 INFO:teuthology.orchestra.run.smithi151.stdout: "mon", 2023-11-16T13:46:02.846 INFO:teuthology.orchestra.run.smithi151.stdout: "nfs", 2023-11-16T13:46:02.846 INFO:teuthology.orchestra.run.smithi151.stdout: "mds", 2023-11-16T13:46:02.846 INFO:teuthology.orchestra.run.smithi151.stdout: "osd", 2023-11-16T13:46:02.846 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:46:02.846 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:46:02.846 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "18/23 daemons upgraded", 2023-11-16T13:46:02.846 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading node-exporter daemons", 2023-11-16T13:46:02.846 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:46:02.846 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:46:03.596 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:03 smithi151 ceph-mon[155575]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:03.596 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:03 smithi151 ceph-mon[155575]: pgmap v242: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:46:03.596 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:03 smithi151 ceph-mon[155575]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:03.596 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:03 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/4095831205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:03.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:03 smithi167 ceph-mon[138945]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:03.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:03 smithi167 ceph-mon[138945]: pgmap v242: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:46:03.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:03 smithi167 ceph-mon[138945]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:03.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:03 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/4095831205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:04.420 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:04 smithi151 ceph-mon[155575]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:04.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:04 smithi167 ceph-mon[138945]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:05.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:05 smithi167 ceph-mon[138945]: pgmap v243: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2023-11-16T13:46:05.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:05 smithi151 ceph-mon[155575]: pgmap v243: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2023-11-16T13:46:07.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:07 smithi167 ceph-mon[138945]: pgmap v244: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:46:07.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:07 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:07.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:07 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:07 smithi151 ceph-mon[155575]: pgmap v244: 129 pgs: 129 active+clean; 223 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2023-11-16T13:46:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:07 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:07 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:09.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:09 smithi167 ceph-mon[138945]: Upgrade: Updating node-exporter.smithi167 (2/2) 2023-11-16T13:46:09.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:09 smithi167 ceph-mon[138945]: Deploying daemon node-exporter.smithi167 on smithi167 2023-11-16T13:46:09.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:09 smithi167 ceph-mon[138945]: pgmap v245: 129 pgs: 129 active+clean; 221 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 9.1 KiB/s wr, 1 op/s 2023-11-16T13:46:09.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:09 smithi151 ceph-mon[155575]: Upgrade: Updating node-exporter.smithi167 (2/2) 2023-11-16T13:46:09.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:09 smithi151 ceph-mon[155575]: Deploying daemon node-exporter.smithi167 on smithi167 2023-11-16T13:46:09.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:09 smithi151 ceph-mon[155575]: pgmap v245: 129 pgs: 129 active+clean; 221 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 9.1 KiB/s wr, 1 op/s 2023-11-16T13:46:11.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:11 smithi167 ceph-mon[138945]: pgmap v246: 129 pgs: 129 active+clean; 220 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 9.2 KiB/s wr, 1 op/s 2023-11-16T13:46:11.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:11.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:11.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:11 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:11.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:11 smithi151 ceph-mon[155575]: pgmap v246: 129 pgs: 129 active+clean; 220 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 9.2 KiB/s wr, 1 op/s 2023-11-16T13:46:11.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:11.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:11.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:11 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:13.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:13 smithi167 ceph-mon[138945]: pgmap v247: 129 pgs: 129 active+clean; 216 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 9.2 KiB/s wr, 2 op/s 2023-11-16T13:46:13.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:13.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:13.598 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:46:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:13 smithi151 ceph-mon[155575]: pgmap v247: 129 pgs: 129 active+clean; 216 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 9.2 KiB/s wr, 2 op/s 2023-11-16T13:46:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:46:14.723 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:14.723 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:14.723 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:14 smithi167 ceph-mon[138945]: pgmap v248: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 597 B/s rd, 11 KiB/s wr, 3 op/s 2023-11-16T13:46:14.723 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:14.724 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:14 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:14.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:14.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:14.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:14 smithi151 ceph-mon[155575]: pgmap v248: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 597 B/s rd, 11 KiB/s wr, 3 op/s 2023-11-16T13:46:14.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:14.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:14 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:16.035 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:16.035 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:15 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:16.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:16.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:15 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:17.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:17 smithi167 ceph-mon[138945]: pgmap v249: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 597 B/s rd, 11 KiB/s wr, 3 op/s 2023-11-16T13:46:17.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:17 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:17.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:17 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:17.456 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:17 smithi151 ceph-mon[155575]: pgmap v249: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 597 B/s rd, 11 KiB/s wr, 3 op/s 2023-11-16T13:46:17.456 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:17 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:17.456 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:17 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:18.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:18.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:18.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:46:18.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:46:18.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:18.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:18.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:18.701 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.702 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.702 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.702 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.702 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.702 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.702 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.702 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.702 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.702 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.702 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.702 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:18 smithi151 ceph-mon[155575]: pgmap v250: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 597 B/s rd, 11 KiB/s wr, 3 op/s 2023-11-16T13:46:18.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:18.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:18.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:46:18.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:46:18.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:18.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:18.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:18 smithi167 ceph-mon[138945]: pgmap v250: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 597 B/s rd, 11 KiB/s wr, 3 op/s 2023-11-16T13:46:19.838 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:19 smithi151 ceph-mon[155575]: Upgrade: Updating prometheus.smithi151 2023-11-16T13:46:19.838 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:19 smithi151 ceph-mon[155575]: Deploying daemon prometheus.smithi151 on smithi151 2023-11-16T13:46:20.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:19 smithi167 ceph-mon[138945]: Upgrade: Updating prometheus.smithi151 2023-11-16T13:46:20.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:19 smithi167 ceph-mon[138945]: Deploying daemon prometheus.smithi151 on smithi151 2023-11-16T13:46:20.753 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:20 smithi151 ceph-mon[155575]: pgmap v251: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 426 B/s rd, 2.2 KiB/s wr, 2 op/s 2023-11-16T13:46:21.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:20 smithi167 ceph-mon[138945]: pgmap v251: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 426 B/s rd, 2.2 KiB/s wr, 2 op/s 2023-11-16T13:46:23.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:22 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:23.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:22 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:23.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:22 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:23.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:22 smithi167 ceph-mon[138945]: pgmap v252: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 597 B/s rd, 2.1 KiB/s wr, 2 op/s 2023-11-16T13:46:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:22 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:22 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:22 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:23.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:22 smithi151 ceph-mon[155575]: pgmap v252: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 597 B/s rd, 2.1 KiB/s wr, 2 op/s 2023-11-16T13:46:25.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:25 smithi151 ceph-mon[155575]: pgmap v253: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 426 B/s rd, 2.1 KiB/s wr, 1 op/s 2023-11-16T13:46:25.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:25.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:25 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:25.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:25 smithi167 ceph-mon[138945]: pgmap v253: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 426 B/s rd, 2.1 KiB/s wr, 1 op/s 2023-11-16T13:46:25.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:25.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:25 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:26.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:26 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:26.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:26 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:26.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:26 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:26.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:26 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:27.656 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:27 smithi151 ceph-mon[155575]: pgmap v254: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2023-11-16T13:46:27.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:27 smithi167 ceph-mon[138945]: pgmap v254: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2023-11-16T13:46:29.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:29.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:29.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:46:29.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:46:29.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:29.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:29.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:46:29.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: pgmap v255: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:29.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-11-16T13:46:29.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-11-16T13:46:29.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:29.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.236 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.237 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: pgmap v255: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:29.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:30.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:29 smithi167 ceph-mon[138945]: Upgrade: Updating alertmanager.smithi151 2023-11-16T13:46:30.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:29 smithi167 ceph-mon[138945]: Deploying daemon alertmanager.smithi151 on smithi151 2023-11-16T13:46:30.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:29 smithi151 ceph-mon[155575]: Upgrade: Updating alertmanager.smithi151 2023-11-16T13:46:30.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:29 smithi151 ceph-mon[155575]: Deploying daemon alertmanager.smithi151 on smithi151 2023-11-16T13:46:31.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:30 smithi151 ceph-mon[155575]: pgmap v256: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:31.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:31 smithi167 ceph-mon[138945]: pgmap v256: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:33.180 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:46:33.370 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:33 smithi151 ceph-mon[155575]: pgmap v257: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:46:33.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:33 smithi167 ceph-mon[138945]: pgmap v257: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:46:33.829 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:46:33.829 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (14m) 9s ago 15m 23.3M - 0.20.0 0881eb8f169f 7fdd307052f6 2023-11-16T13:46:33.830 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (6m) 9s ago 15m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:46:33.830 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (6m) 20s ago 15m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:46:33.830 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (14m) 9s ago 15m 37.0M - 6.7.4 557c83e11646 a98058834c8f 2023-11-16T13:46:33.830 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (2m) 9s ago 12m 46.1M - 17.2.7-68-gae6ce92c ea201341e8f3 811376ceeaa9 2023-11-16T13:46:33.830 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (2m) 20s ago 12m 13.4M - 17.2.7-68-gae6ce92c ea201341e8f3 36ac20a06016 2023-11-16T13:46:33.830 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (7m) 9s ago 16m 519M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:46:33.830 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (7m) 20s ago 15m 440M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:46:33.830 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (7m) 9s ago 17m 116M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:46:33.830 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (6m) 20s ago 15m 84.6M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:46:33.830 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (2m) 9s ago 10m 110M - 4.0 ea201341e8f3 4d8e7f0525d7 2023-11-16T13:46:33.830 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (2m) 20s ago 9m 59.2M - 4.0 ea201341e8f3 6b233b631706 2023-11-16T13:46:33.831 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (27s) 9s ago 15m 8753k - 1.5.0 0da6a335fe13 752eecf50e92 2023-11-16T13:46:33.831 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (23s) 20s ago 14m 4140k - 1.5.0 0da6a335fe13 cc63c9bab829 2023-11-16T13:46:33.831 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (6m) 9s ago 14m 453M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:46:33.831 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (5m) 9s ago 14m 405M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 49b15d0ef2d2 2023-11-16T13:46:33.831 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (5m) 9s ago 14m 322M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 89fcacc46719 2023-11-16T13:46:33.831 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (4m) 9s ago 13m 428M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 aa3cf87e766a 2023-11-16T13:46:33.831 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (4m) 20s ago 13m 428M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 cbb1a9fdea87 2023-11-16T13:46:33.831 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (3m) 20s ago 13m 420M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 82902ee5659d 2023-11-16T13:46:33.831 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (3m) 20s ago 13m 331M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 6c4e2d28bf30 2023-11-16T13:46:33.831 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (2m) 20s ago 13m 494M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 8abcdef4b8d0 2023-11-16T13:46:33.831 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (12s) 9s ago 15m 49.8M - 2.43.0 a07b618ecd1d 3a4f0167c8ae 2023-11-16T13:46:34.240 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:46:34.240 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:46:34.241 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:46:34.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:46:34.241 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:46:34.241 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:46:34.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:46:34.241 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:46:34.241 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 8 2023-11-16T13:46:34.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:46:34.241 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:46:34.241 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:46:34.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:46:34.242 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:46:34.242 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 14 2023-11-16T13:46:34.242 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:46:34.242 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:46:34.293 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:34 smithi151 ceph-mon[155575]: from='client.34460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:34.293 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:34 smithi151 ceph-mon[155575]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:34.294 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:34 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/1966893965' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:34.593 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:46:34.593 INFO:teuthology.orchestra.run.smithi151.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", 2023-11-16T13:46:34.593 INFO:teuthology.orchestra.run.smithi151.stdout: "in_progress": true, 2023-11-16T13:46:34.593 INFO:teuthology.orchestra.run.smithi151.stdout: "which": "Upgrading all daemon types on all hosts", 2023-11-16T13:46:34.593 INFO:teuthology.orchestra.run.smithi151.stdout: "services_complete": [ 2023-11-16T13:46:34.593 INFO:teuthology.orchestra.run.smithi151.stdout: "crash", 2023-11-16T13:46:34.593 INFO:teuthology.orchestra.run.smithi151.stdout: "mon", 2023-11-16T13:46:34.594 INFO:teuthology.orchestra.run.smithi151.stdout: "nfs", 2023-11-16T13:46:34.594 INFO:teuthology.orchestra.run.smithi151.stdout: "mds", 2023-11-16T13:46:34.594 INFO:teuthology.orchestra.run.smithi151.stdout: "osd", 2023-11-16T13:46:34.594 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr" 2023-11-16T13:46:34.594 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:46:34.594 INFO:teuthology.orchestra.run.smithi151.stdout: "progress": "18/23 daemons upgraded", 2023-11-16T13:46:34.594 INFO:teuthology.orchestra.run.smithi151.stdout: "message": "Currently upgrading alertmanager daemons", 2023-11-16T13:46:34.594 INFO:teuthology.orchestra.run.smithi151.stdout: "is_paused": false 2023-11-16T13:46:34.594 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:46:34.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:34 smithi167 ceph-mon[138945]: from='client.34460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:34.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:34 smithi167 ceph-mon[138945]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:34.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:34 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/1966893965' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:35.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:35 smithi167 ceph-mon[138945]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:35.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:35 smithi167 ceph-mon[138945]: pgmap v258: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:35.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:35 smithi151 ceph-mon[155575]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:35.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:35 smithi151 ceph-mon[155575]: pgmap v258: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:36.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:36 smithi167 ceph-mon[138945]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:36.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:36 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:36.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:36 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:36.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:36 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:36.977 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:36 smithi151 ceph-mon[155575]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:46:36.978 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:36 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:36.978 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:36 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:36.978 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:36 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:37.718 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:37 smithi151 ceph-mon[155575]: pgmap v259: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:37.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:37 smithi167 ceph-mon[138945]: pgmap v259: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:39.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:39 smithi151 ceph-mon[155575]: pgmap v260: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:39.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:39.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:39.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:39.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:39 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:39.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:39 smithi167 ceph-mon[138945]: pgmap v260: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:39.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:39.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:39.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:39.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:39 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:41.725 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:41 smithi151 ceph-mon[155575]: pgmap v261: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:41.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:41 smithi167 ceph-mon[138945]: pgmap v261: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:43.106 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:43.106 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:43.106 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:46:43.106 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:46:43.106 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:43.106 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:43.106 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: pgmap v262: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:46:43.106 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-11-16T13:46:43.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-11-16T13:46:43.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi151.front.sepia.ceph.com:9093"}]: dispatch 2023-11-16T13:46:43.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi151.front.sepia.ceph.com:9093"}]: dispatch 2023-11-16T13:46:43.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:43.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:43.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.107 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.108 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:42 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:43.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:43.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:46:43.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:46:43.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:43.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:43.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: pgmap v262: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:46:43.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-11-16T13:46:43.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-11-16T13:46:43.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi151.front.sepia.ceph.com:9093"}]: dispatch 2023-11-16T13:46:43.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi151.front.sepia.ceph.com:9093"}]: dispatch 2023-11-16T13:46:43.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:43.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:43.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.348 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:43.349 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:42 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:46:44.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:46:44.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:43 smithi151 ceph-mon[155575]: Upgrade: Updating grafana.smithi151 2023-11-16T13:46:44.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:44.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:44.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-11-16T13:46:44.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:43 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-11-16T13:46:44.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:43 smithi151 ceph-mon[155575]: Deploying daemon grafana.smithi151 on smithi151 2023-11-16T13:46:44.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:46:44.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:43 smithi167 ceph-mon[138945]: Upgrade: Updating grafana.smithi151 2023-11-16T13:46:44.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:44.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:44.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-11-16T13:46:44.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:43 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-11-16T13:46:44.347 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:43 smithi167 ceph-mon[138945]: Deploying daemon grafana.smithi151 on smithi151 2023-11-16T13:46:45.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:44 smithi151 ceph-mon[155575]: pgmap v263: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:45.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:44 smithi167 ceph-mon[138945]: pgmap v263: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:47.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:47 smithi167 ceph-mon[138945]: pgmap v264: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:47.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:47 smithi151 ceph-mon[155575]: pgmap v264: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:49.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:49 smithi167 ceph-mon[138945]: pgmap v265: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:49.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:49 smithi151 ceph-mon[155575]: pgmap v265: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:51.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:51 smithi167 ceph-mon[138945]: pgmap v266: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:51.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:51 smithi151 ceph-mon[155575]: pgmap v266: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:53.441 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:53 smithi151 ceph-mon[155575]: pgmap v267: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:46:53.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:53 smithi167 ceph-mon[138945]: pgmap v267: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:46:55.383 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:55 smithi151 ceph-mon[155575]: pgmap v268: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:55.384 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:55 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:55.384 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:55 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:55.384 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:55 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:55.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:55 smithi167 ceph-mon[138945]: pgmap v268: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:55.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:55 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:55.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:55 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:55.597 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:55 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:46:57.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:57 smithi167 ceph-mon[138945]: pgmap v269: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:57 smithi151 ceph-mon[155575]: pgmap v269: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:58.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:58 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:46:58.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:58 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:58.346 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:58 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:58.660 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:58 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:46:58.661 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:58 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:58.661 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:58 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:59 smithi151 ceph-mon[155575]: pgmap v270: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:59.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:46:59 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:59.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:59 smithi167 ceph-mon[138945]: pgmap v270: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:46:59.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:46:59.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:46:59 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:01.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:01 smithi151 ceph-mon[155575]: pgmap v271: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:01.596 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:01 smithi167 ceph-mon[138945]: pgmap v271: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:02.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.735 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.736 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2023-11-16T13:47:02.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2023-11-16T13:47:02.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2023-11-16T13:47:02.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2023-11-16T13:47:02.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2023-11-16T13:47:02.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2023-11-16T13:47:02.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2023-11-16T13:47:02.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2023-11-16T13:47:02.737 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2023-11-16T13:47:02.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2023-11-16T13:47:02.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2023-11-16T13:47:02.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2023-11-16T13:47:02.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2023-11-16T13:47:02.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2023-11-16T13:47:02.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2023-11-16T13:47:02.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2023-11-16T13:47:02.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2023-11-16T13:47:02.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.738 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2023-11-16T13:47:02.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2023-11-16T13:47:02.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2023-11-16T13:47:02.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2023-11-16T13:47:02.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:47:02.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:47:02.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:47:02.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.739 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.740 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:47:02.740 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:47:02.740 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:47:02.740 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.740 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:47:02.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:47:02.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-11-16T13:47:02.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-11-16T13:47:02.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2023-11-16T13:47:02.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2023-11-16T13:47:02.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:47:02.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.848 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:02.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2023-11-16T13:47:02.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2023-11-16T13:47:02.850 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.850 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2023-11-16T13:47:02.850 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2023-11-16T13:47:02.850 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2023-11-16T13:47:02.850 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2023-11-16T13:47:02.850 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2023-11-16T13:47:02.850 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2023-11-16T13:47:02.850 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2023-11-16T13:47:02.850 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2023-11-16T13:47:02.850 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2023-11-16T13:47:02.850 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2023-11-16T13:47:02.850 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2023-11-16T13:47:02.851 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.851 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2023-11-16T13:47:02.851 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2023-11-16T13:47:02.851 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2023-11-16T13:47:02.851 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2023-11-16T13:47:02.851 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.851 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.851 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.851 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.851 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.851 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2023-11-16T13:47:02.852 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2023-11-16T13:47:02.852 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2023-11-16T13:47:02.852 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2023-11-16T13:47:02.852 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2023-11-16T13:47:02.852 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:47:02.852 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:47:02.852 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:47:02.852 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.852 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.852 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:47:02.852 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:47:02.852 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:47:02.853 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:02.853 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:03 smithi151 ceph-mon[155575]: Upgrade: Finalizing container_image settings 2023-11-16T13:47:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:03 smithi151 ceph-mon[155575]: Upgrade: Complete! 2023-11-16T13:47:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:03 smithi151 ceph-mon[155575]: pgmap v272: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:47:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:03 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:03.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:03 smithi167 ceph-mon[138945]: Upgrade: Finalizing container_image settings 2023-11-16T13:47:03.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:03 smithi167 ceph-mon[138945]: Upgrade: Complete! 2023-11-16T13:47:03.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:03 smithi167 ceph-mon[138945]: pgmap v272: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:47:03.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:03 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:47:05.240 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'ceph orch ps' 2023-11-16T13:47:05.484 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:05 smithi151 ceph-mon[155575]: pgmap v273: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:05.849 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:05 smithi167 ceph-mon[138945]: pgmap v273: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:06.630 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:06 smithi151 ceph-mon[155575]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:47:06.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:06 smithi167 ceph-mon[138945]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:47:07.543 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:47:07.543 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (32s) 9s ago 16m 17.7M - 0.25.0 c8568f914cd2 18688d04a923 2023-11-16T13:47:07.543 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (7m) 9s ago 16m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:47:07.543 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (7m) 54s ago 15m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:47:07.543 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (12s) 9s ago 16m 34.0M - 9.4.7 2c41d148cca3 8f05509bb9bb 2023-11-16T13:47:07.543 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (3m) 9s ago 12m 46.1M - 17.2.7-68-gae6ce92c ea201341e8f3 811376ceeaa9 2023-11-16T13:47:07.544 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (3m) 54s ago 12m 13.4M - 17.2.7-68-gae6ce92c ea201341e8f3 36ac20a06016 2023-11-16T13:47:07.544 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (8m) 9s ago 17m 521M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:47:07.544 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (7m) 54s ago 15m 440M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:47:07.544 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (7m) 9s ago 17m 118M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:47:07.544 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (7m) 54s ago 15m 84.6M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:47:07.544 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (2m) 9s ago 10m 112M - 4.0 ea201341e8f3 4d8e7f0525d7 2023-11-16T13:47:07.544 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (2m) 54s ago 10m 59.2M - 4.0 ea201341e8f3 6b233b631706 2023-11-16T13:47:07.544 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (60s) 9s ago 15m 9457k - 1.5.0 0da6a335fe13 752eecf50e92 2023-11-16T13:47:07.544 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (56s) 54s ago 15m 4140k - 1.5.0 0da6a335fe13 cc63c9bab829 2023-11-16T13:47:07.544 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (6m) 9s ago 15m 455M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:47:07.544 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (6m) 9s ago 14m 407M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 49b15d0ef2d2 2023-11-16T13:47:07.544 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (5m) 9s ago 14m 323M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 89fcacc46719 2023-11-16T13:47:07.544 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (5m) 9s ago 14m 431M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 aa3cf87e766a 2023-11-16T13:47:07.545 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (4m) 54s ago 14m 428M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 cbb1a9fdea87 2023-11-16T13:47:07.545 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (4m) 54s ago 14m 420M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 82902ee5659d 2023-11-16T13:47:07.545 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (3m) 54s ago 13m 331M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 6c4e2d28bf30 2023-11-16T13:47:07.545 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (3m) 54s ago 13m 494M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 8abcdef4b8d0 2023-11-16T13:47:07.545 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (46s) 9s ago 15m 57.1M - 2.43.0 a07b618ecd1d 3a4f0167c8ae 2023-11-16T13:47:07.720 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:07 smithi151 ceph-mon[155575]: pgmap v274: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:07.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:07 smithi167 ceph-mon[138945]: pgmap v274: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:07.924 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'ceph versions' 2023-11-16T13:47:09.298 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:47:09.299 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:47:09.299 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:47:09.299 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:47:09.299 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:47:09.299 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:47:09.299 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:47:09.299 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:47:09.299 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 8 2023-11-16T13:47:09.299 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:47:09.299 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:47:09.299 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:47:09.300 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:47:09.300 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:47:09.300 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 14 2023-11-16T13:47:09.300 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:47:09.300 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:47:09.573 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:09 smithi151 ceph-mon[155575]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:47:09.574 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:09 smithi151 ceph-mon[155575]: pgmap v275: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:09.574 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:09 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/2173803674' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:09.658 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2023-11-16T13:47:09.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:09 smithi167 ceph-mon[138945]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:47:09.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:09 smithi167 ceph-mon[138945]: pgmap v275: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:09.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:09 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/2173803674' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:47:10.720 INFO:teuthology.orchestra.run.smithi151.stdout:wait for servicemap items w/ changing names to refresh 2023-11-16T13:47:11.058 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'sleep 60' 2023-11-16T13:47:11.678 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:11 smithi151 ceph-mon[155575]: pgmap v276: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:11.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:11 smithi167 ceph-mon[138945]: pgmap v276: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:13 smithi151 ceph-mon[155575]: pgmap v277: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:47:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:47:13.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:13 smithi167 ceph-mon[138945]: pgmap v277: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:47:13.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:47:15.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:15 smithi151 ceph-mon[155575]: pgmap v278: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:15.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:15 smithi167 ceph-mon[138945]: pgmap v278: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:17.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:17 smithi151 ceph-mon[155575]: pgmap v279: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:17 smithi167 ceph-mon[138945]: pgmap v279: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:19.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:19 smithi151 ceph-mon[155575]: pgmap v280: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:19.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:19 smithi167 ceph-mon[138945]: pgmap v280: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:21.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:21 smithi151 ceph-mon[155575]: pgmap v281: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:21.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:21 smithi167 ceph-mon[138945]: pgmap v281: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:23.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:23 smithi151 ceph-mon[155575]: pgmap v282: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:47:23.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:23 smithi167 ceph-mon[138945]: pgmap v282: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:47:25.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:25 smithi151 ceph-mon[155575]: pgmap v283: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:25.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:25 smithi167 ceph-mon[138945]: pgmap v283: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:27.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:27 smithi151 ceph-mon[155575]: pgmap v284: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:27.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:27 smithi167 ceph-mon[138945]: pgmap v284: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:28.661 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:28 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:47:28.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:28 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:47:29.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:29 smithi151 ceph-mon[155575]: pgmap v285: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:29.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:29 smithi167 ceph-mon[138945]: pgmap v285: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:31.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:31 smithi151 ceph-mon[155575]: pgmap v286: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:35.299 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:33 smithi151 ceph-mon[155575]: pgmap v287: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:47:35.299 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:31 smithi167 ceph-mon[138945]: pgmap v286: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:35.299 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:33 smithi167 ceph-mon[138945]: pgmap v287: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:47:35.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:35 smithi151 ceph-mon[155575]: pgmap v288: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:35.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:35 smithi167 ceph-mon[138945]: pgmap v288: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:37.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:37 smithi151 ceph-mon[155575]: pgmap v289: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:37.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:37 smithi167 ceph-mon[138945]: pgmap v289: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:39.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:39 smithi151 ceph-mon[155575]: pgmap v290: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:39.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:39 smithi167 ceph-mon[138945]: pgmap v290: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:41.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:41 smithi151 ceph-mon[155575]: pgmap v291: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:41.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:41 smithi167 ceph-mon[138945]: pgmap v291: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:43.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:43 smithi151 ceph-mon[155575]: pgmap v292: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:47:43.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:43 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:47:43.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:43 smithi167 ceph-mon[138945]: pgmap v292: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:47:43.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:43 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:47:45.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:45 smithi151 ceph-mon[155575]: pgmap v293: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:45.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:45 smithi167 ceph-mon[138945]: pgmap v293: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:47.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:47 smithi151 ceph-mon[155575]: pgmap v294: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:47.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:47 smithi167 ceph-mon[138945]: pgmap v294: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:49.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:49 smithi151 ceph-mon[155575]: pgmap v295: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:49.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:49 smithi167 ceph-mon[138945]: pgmap v295: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:51.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:51 smithi151 ceph-mon[155575]: pgmap v296: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:51.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:51 smithi167 ceph-mon[138945]: pgmap v296: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:53.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:53 smithi151 ceph-mon[155575]: pgmap v297: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:47:53.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:53 smithi167 ceph-mon[138945]: pgmap v297: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:47:55.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:55 smithi151 ceph-mon[155575]: pgmap v298: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:55.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:55 smithi167 ceph-mon[138945]: pgmap v298: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:57.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:57 smithi151 ceph-mon[155575]: pgmap v299: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:57.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:57 smithi167 ceph-mon[138945]: pgmap v299: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:58.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:58 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:47:58.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:58 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:47:59.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:47:59 smithi151 ceph-mon[155575]: pgmap v300: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:47:59.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:47:59 smithi167 ceph-mon[138945]: pgmap v300: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:01.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:01 smithi151 ceph-mon[155575]: pgmap v301: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:01.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:01 smithi167 ceph-mon[138945]: pgmap v301: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:02.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:48:02.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:48:02.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:48:02.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:48:02.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:02 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:48:02.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-16T13:48:02.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-16T13:48:02.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-11-16T13:48:02.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:48:02.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:02 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' 2023-11-16T13:48:03.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:03 smithi151 ceph-mon[155575]: pgmap v302: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:48:03.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:03 smithi167 ceph-mon[138945]: pgmap v302: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:48:05.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:05 smithi151 ceph-mon[155575]: pgmap v303: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:05.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:05 smithi167 ceph-mon[138945]: pgmap v303: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:07.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:07 smithi151 ceph-mon[155575]: pgmap v304: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:07.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:07 smithi167 ceph-mon[138945]: pgmap v304: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:09.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:09 smithi151 ceph-mon[155575]: pgmap v305: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:09.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:09 smithi167 ceph-mon[138945]: pgmap v305: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:11.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:11 smithi151 ceph-mon[155575]: pgmap v306: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:11.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:11 smithi167 ceph-mon[138945]: pgmap v306: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:12.411 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'ceph orch ps' 2023-11-16T13:48:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:13 smithi151 ceph-mon[155575]: pgmap v307: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:48:13.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:13 smithi151 ceph-mon[155575]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:48:13.753 INFO:teuthology.orchestra.run.smithi151.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-11-16T13:48:13.753 INFO:teuthology.orchestra.run.smithi151.stdout:alertmanager.smithi151 smithi151 *:9093,9094 running (98s) 75s ago 17m 17.7M - 0.25.0 c8568f914cd2 18688d04a923 2023-11-16T13:48:13.753 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi151 smithi151 running (8m) 75s ago 17m 7482k - 17.2.7-68-gae6ce92c ea201341e8f3 95471cf6552b 2023-11-16T13:48:13.753 INFO:teuthology.orchestra.run.smithi151.stdout:crash.smithi167 smithi167 running (8m) 2m ago 16m 7432k - 17.2.7-68-gae6ce92c ea201341e8f3 00d42804f95d 2023-11-16T13:48:13.753 INFO:teuthology.orchestra.run.smithi151.stdout:grafana.smithi151 smithi151 *:3000 running (79s) 75s ago 17m 34.0M - 9.4.7 2c41d148cca3 8f05509bb9bb 2023-11-16T13:48:13.753 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi151.mdkcfw smithi151 running (4m) 75s ago 13m 46.1M - 17.2.7-68-gae6ce92c ea201341e8f3 811376ceeaa9 2023-11-16T13:48:13.753 INFO:teuthology.orchestra.run.smithi151.stdout:mds.foofs.smithi167.irznoa smithi167 running (4m) 2m ago 13m 13.4M - 17.2.7-68-gae6ce92c ea201341e8f3 36ac20a06016 2023-11-16T13:48:13.753 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi151.fcujra smithi151 *:8443,9283 running (9m) 75s ago 18m 521M - 17.2.7-68-gae6ce92c ea201341e8f3 31ff6d226fb5 2023-11-16T13:48:13.753 INFO:teuthology.orchestra.run.smithi151.stdout:mgr.smithi167.xxnzsz smithi167 *:8443,9283 running (9m) 2m ago 16m 440M - 17.2.7-68-gae6ce92c ea201341e8f3 aba6702496c4 2023-11-16T13:48:13.753 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi151 smithi151 running (8m) 75s ago 18m 118M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 77b2d923be97 2023-11-16T13:48:13.754 INFO:teuthology.orchestra.run.smithi151.stdout:mon.smithi167 smithi167 running (8m) 2m ago 16m 84.6M 2048M 17.2.7-68-gae6ce92c ea201341e8f3 1176c78880ed 2023-11-16T13:48:13.754 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.0.1.smithi151.qfkjic smithi151 *:2049 running (4m) 75s ago 11m 112M - 4.0 ea201341e8f3 4d8e7f0525d7 2023-11-16T13:48:13.754 INFO:teuthology.orchestra.run.smithi151.stdout:nfs.foo.1.1.smithi167.rpoanb smithi167 *:2049 running (3m) 2m ago 11m 59.2M - 4.0 ea201341e8f3 6b233b631706 2023-11-16T13:48:13.754 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi151 smithi151 *:9100 running (2m) 75s ago 17m 9457k - 1.5.0 0da6a335fe13 752eecf50e92 2023-11-16T13:48:13.754 INFO:teuthology.orchestra.run.smithi151.stdout:node-exporter.smithi167 smithi167 *:9100 running (2m) 2m ago 16m 4140k - 1.5.0 0da6a335fe13 cc63c9bab829 2023-11-16T13:48:13.754 INFO:teuthology.orchestra.run.smithi151.stdout:osd.0 smithi151 running (7m) 75s ago 16m 455M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 66e6cf45601a 2023-11-16T13:48:13.754 INFO:teuthology.orchestra.run.smithi151.stdout:osd.1 smithi151 running (7m) 75s ago 16m 407M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 49b15d0ef2d2 2023-11-16T13:48:13.754 INFO:teuthology.orchestra.run.smithi151.stdout:osd.2 smithi151 running (6m) 75s ago 15m 323M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 89fcacc46719 2023-11-16T13:48:13.754 INFO:teuthology.orchestra.run.smithi151.stdout:osd.3 smithi151 running (6m) 75s ago 15m 431M 1944M 17.2.7-68-gae6ce92c ea201341e8f3 aa3cf87e766a 2023-11-16T13:48:13.754 INFO:teuthology.orchestra.run.smithi151.stdout:osd.4 smithi167 running (5m) 2m ago 15m 428M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 cbb1a9fdea87 2023-11-16T13:48:13.754 INFO:teuthology.orchestra.run.smithi151.stdout:osd.5 smithi167 running (5m) 2m ago 15m 420M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 82902ee5659d 2023-11-16T13:48:13.754 INFO:teuthology.orchestra.run.smithi151.stdout:osd.6 smithi167 running (5m) 2m ago 14m 331M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 6c4e2d28bf30 2023-11-16T13:48:13.755 INFO:teuthology.orchestra.run.smithi151.stdout:osd.7 smithi167 running (4m) 2m ago 14m 494M 2712M 17.2.7-68-gae6ce92c ea201341e8f3 8abcdef4b8d0 2023-11-16T13:48:13.755 INFO:teuthology.orchestra.run.smithi151.stdout:prometheus.smithi151 smithi151 *:9095 running (112s) 75s ago 16m 57.1M - 2.43.0 a07b618ecd1d 3a4f0167c8ae 2023-11-16T13:48:13.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:13 smithi167 ceph-mon[138945]: pgmap v307: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:48:13.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:13 smithi167 ceph-mon[138945]: from='mgr.34102 172.21.15.151:0/3120569575' entity='mgr.smithi151.fcujra' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-11-16T13:48:14.122 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'ceph versions' 2023-11-16T13:48:15.509 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-11-16T13:48:15.509 INFO:teuthology.orchestra.run.smithi151.stdout: "mon": { 2023-11-16T13:48:15.509 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:48:15.509 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:48:15.509 INFO:teuthology.orchestra.run.smithi151.stdout: "mgr": { 2023-11-16T13:48:15.509 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:48:15.509 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:48:15.509 INFO:teuthology.orchestra.run.smithi151.stdout: "osd": { 2023-11-16T13:48:15.509 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 8 2023-11-16T13:48:15.509 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:48:15.509 INFO:teuthology.orchestra.run.smithi151.stdout: "mds": { 2023-11-16T13:48:15.510 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 2 2023-11-16T13:48:15.510 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:48:15.510 INFO:teuthology.orchestra.run.smithi151.stdout: "overall": { 2023-11-16T13:48:15.510 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)": 14 2023-11-16T13:48:15.510 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:48:15.510 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-11-16T13:48:15.779 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:15 smithi151 ceph-mon[155575]: from='client.34492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:48:15.779 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:15 smithi151 ceph-mon[155575]: pgmap v308: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:15.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:15 smithi167 ceph-mon[138945]: from='client.34492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:48:15.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:15 smithi167 ceph-mon[138945]: pgmap v308: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:15.880 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2023-11-16T13:48:16.826 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:16 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/1630268160' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:48:16.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:16 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/1630268160' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:48:17.284 INFO:teuthology.orchestra.run.smithi151.stdout:true 2023-11-16T13:48:17.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:17 smithi151 ceph-mon[155575]: pgmap v309: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:17.554 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:17 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/1295432576' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:48:17.655 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -e sha1=ae6ce92c0812ba06c9c623ce72df4f3fc59080b4 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2023-11-16T13:48:17.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:17 smithi167 ceph-mon[138945]: pgmap v309: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:17.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:17 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/1295432576' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:48:19.061 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph version 17.2.7-68-gae6ce92c (ae6ce92c0812ba06c9c623ce72df4f3fc59080b4) quincy (stable)" 2023-11-16T13:48:19.406 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2023-11-16T13:48:19.406 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2023-11-16T13:48:19.407 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- ceph orch ls -f json 2023-11-16T13:48:19.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:19 smithi151 ceph-mon[155575]: pgmap v310: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:19.734 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:19 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/81495875' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:48:19.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:19 smithi167 ceph-mon[138945]: pgmap v310: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:19.847 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:19 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/81495875' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-11-16T13:48:20.743 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-11-16T13:48:20.743 INFO:teuthology.orchestra.run.smithi151.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2023-11-16T13:30:13.774692Z", "last_refresh": "2023-11-16T13:46:58.064399Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2023-11-16T13:30:03.539779Z", "last_refresh": "2023-11-16T13:46:12.896195Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true}, "status": {"created": "2023-11-16T13:30:12.037972Z", "last_refresh": "2023-11-16T13:46:58.064527Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2023-11-16T13:34:23.940012Z", "last_refresh": "2023-11-16T13:46:12.896988Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2023-11-16T13:30:02.671949Z", "last_refresh": "2023-11-16T13:46:12.896337Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi151:172.21.15.151=smithi151", "smithi167:172.21.15.167=smithi167"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2023-11-16T13:30:38.525496Z", "last_refresh": "2023-11-16T13:46:12.896424Z", "running": 2, "size": 2}}, {"events": ["2023-11-16T13:39:35.070971Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "status": {"created": "2023-11-16T13:36:30.739527Z", "last_refresh": "2023-11-16T13:46:12.897092Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2023-11-16T13:30:12.907869Z", "last_refresh": "2023-11-16T13:46:12.896490Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "ea201341e8f356dcfb0398db2ba62378ab4c4eb62a6f18a246840cbf8e818dc5", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:95ba1944a7125cdac5ea33490979633c0bd423bb84277aa901268e12e0c9863b", "last_refresh": "2023-11-16T13:46:12.896568Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2023-11-16T13:30:11.164903Z", "last_refresh": "2023-11-16T13:46:58.064676Z", "ports": [9095], "running": 1, "size": 1}}] 2023-11-16T13:48:21.121 INFO:tasks.cephadm:nfs.foo has 2/2 2023-11-16T13:48:21.121 DEBUG:teuthology.parallel:result is None 2023-11-16T13:48:21.121 INFO:teuthology.run_tasks:Running task vip.exec... 2023-11-16T13:48:21.132 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi151.front.sepia.ceph.com 2023-11-16T13:48:21.132 DEBUG:teuthology.orchestra.run.smithi151:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2023-11-16T13:48:21.162 INFO:teuthology.orchestra.run.smithi151.stderr:+ umount /mnt/foo 2023-11-16T13:48:21.197 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-11-16T13:48:21.208 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi151.front.sepia.ceph.com 2023-11-16T13:48:21.209 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- bash -c 'ceph nfs cluster ls | grep foo' 2023-11-16T13:48:21.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:21 smithi167 ceph-mon[138945]: pgmap v311: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:21.984 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:21 smithi151 ceph-mon[155575]: pgmap v311: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:22.581 INFO:teuthology.orchestra.run.smithi151.stdout:foo 2023-11-16T13:48:22.828 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:22 smithi151 ceph-mon[155575]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-11-16T13:48:22.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:22 smithi167 ceph-mon[138945]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-11-16T13:48:22.938 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- bash -c 'ceph nfs export ls foo --detailed' 2023-11-16T13:48:23.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:23 smithi167 ceph-mon[138945]: pgmap v312: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:48:23.892 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:23 smithi151 ceph-mon[155575]: pgmap v312: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2023-11-16T13:48:24.320 INFO:teuthology.orchestra.run.smithi151.stdout:[ 2023-11-16T13:48:24.320 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-11-16T13:48:24.320 INFO:teuthology.orchestra.run.smithi151.stdout: "export_id": 1, 2023-11-16T13:48:24.320 INFO:teuthology.orchestra.run.smithi151.stdout: "path": "/", 2023-11-16T13:48:24.320 INFO:teuthology.orchestra.run.smithi151.stdout: "cluster_id": "foo", 2023-11-16T13:48:24.321 INFO:teuthology.orchestra.run.smithi151.stdout: "pseudo": "/fake", 2023-11-16T13:48:24.321 INFO:teuthology.orchestra.run.smithi151.stdout: "access_type": "RW", 2023-11-16T13:48:24.321 INFO:teuthology.orchestra.run.smithi151.stdout: "squash": "no_root_squash", 2023-11-16T13:48:24.321 INFO:teuthology.orchestra.run.smithi151.stdout: "security_label": true, 2023-11-16T13:48:24.321 INFO:teuthology.orchestra.run.smithi151.stdout: "protocols": [ 2023-11-16T13:48:24.321 INFO:teuthology.orchestra.run.smithi151.stdout: 4 2023-11-16T13:48:24.321 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:48:24.321 INFO:teuthology.orchestra.run.smithi151.stdout: "transports": [ 2023-11-16T13:48:24.321 INFO:teuthology.orchestra.run.smithi151.stdout: "TCP" 2023-11-16T13:48:24.321 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-11-16T13:48:24.321 INFO:teuthology.orchestra.run.smithi151.stdout: "fsal": { 2023-11-16T13:48:24.321 INFO:teuthology.orchestra.run.smithi151.stdout: "name": "CEPH", 2023-11-16T13:48:24.322 INFO:teuthology.orchestra.run.smithi151.stdout: "user_id": "nfs.foo.1", 2023-11-16T13:48:24.322 INFO:teuthology.orchestra.run.smithi151.stdout: "fs_name": "foofs" 2023-11-16T13:48:24.322 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-11-16T13:48:24.322 INFO:teuthology.orchestra.run.smithi151.stdout: "clients": [] 2023-11-16T13:48:24.322 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-11-16T13:48:24.322 INFO:teuthology.orchestra.run.smithi151.stdout:] 2023-11-16T13:48:24.695 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:24 smithi151 ceph-mon[155575]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:48:24.696 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- bash -c 'rados -p .nfs --all ls -' 2023-11-16T13:48:24.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:24 smithi167 ceph-mon[138945]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:48:25.616 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:25 smithi151 ceph-mon[155575]: pgmap v313: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:25.616 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:25 smithi151 ceph-mon[155575]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:48:25.743 INFO:teuthology.orchestra.run.smithi151.stdout:foo grace 2023-11-16T13:48:25.743 INFO:teuthology.orchestra.run.smithi151.stdout:foo rec-0000000000000005:nfs.foo.1 2023-11-16T13:48:25.743 INFO:teuthology.orchestra.run.smithi151.stdout:foo rec-0000000000000005:nfs.foo.0 2023-11-16T13:48:25.743 INFO:teuthology.orchestra.run.smithi151.stdout:foo conf-nfs.foo 2023-11-16T13:48:25.744 INFO:teuthology.orchestra.run.smithi151.stdout:foo export-1 2023-11-16T13:48:25.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:25 smithi167 ceph-mon[138945]: pgmap v313: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-11-16T13:48:25.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:25 smithi167 ceph-mon[138945]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2023-11-16T13:48:26.104 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 -- bash -c 'ceph config get mgr mgr/cephadm/migration_current | grep 5' 2023-11-16T13:48:26.724 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:26 smithi151 ceph-mon[155575]: mgrmap e41: smithi151.fcujra(active, since 8m), standbys: smithi167.xxnzsz 2023-11-16T13:48:26.846 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:26 smithi167 ceph-mon[138945]: mgrmap e41: smithi151.fcujra(active, since 8m), standbys: smithi167.xxnzsz 2023-11-16T13:48:27.406 INFO:teuthology.orchestra.run.smithi151.stdout:5 2023-11-16T13:48:27.669 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:27 smithi151 ceph-mon[155575]: pgmap v314: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-11-16T13:48:27.669 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:27 smithi151 ceph-mon[155575]: from='client.? 172.21.15.151:0/668001906' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2023-11-16T13:48:27.762 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-11-16T13:48:27.774 INFO:tasks.cephadm:Teardown begin 2023-11-16T13:48:27.774 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-11-16T13:48:27.804 DEBUG:teuthology.orchestra.run.smithi167:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-11-16T13:48:27.838 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:27 smithi167 ceph-mon[138945]: pgmap v314: 129 pgs: 129 active+clean; 214 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-11-16T13:48:27.838 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:27 smithi167 ceph-mon[138945]: from='client.? 172.21.15.151:0/668001906' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2023-11-16T13:48:27.843 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-11-16T13:48:27.843 DEBUG:teuthology.orchestra.run.smithi151:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-11-16T13:48:27.862 DEBUG:teuthology.orchestra.run.smithi167:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-11-16T13:48:27.904 INFO:tasks.cephadm:Stopping all daemons... 2023-11-16T13:48:27.905 INFO:tasks.cephadm.mon.smithi151:Stopping mon.smithi151... 2023-11-16T13:48:27.905 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi151 2023-11-16T13:48:28.234 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:27 smithi151 systemd[1]: Stopping Ceph mon.smithi151 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:48:28.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon-smithi151[155552]: 2023-11-16T13:48:28.138+0000 7fa8097e0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi151 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-11-16T13:48:28.235 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:28 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon-smithi151[155552]: 2023-11-16T13:48:28.138+0000 7fa8097e0700 -1 mon.smithi151@0(leader) e3 *** Got Signal Terminated *** 2023-11-16T13:48:28.577 INFO:journalctl@ceph.mon.smithi151.smithi151.stdout:Nov 16 13:48:28 smithi151 bash[213298]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon-smithi151 2023-11-16T13:48:28.584 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi151.service' 2023-11-16T13:48:28.632 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-11-16T13:48:28.632 INFO:tasks.cephadm.mon.smithi151:Stopped mon.smithi151 2023-11-16T13:48:28.633 INFO:tasks.cephadm.mon.smithi167:Stopping mon.smithi167... 2023-11-16T13:48:28.633 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi167 2023-11-16T13:48:29.096 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:28 smithi167 systemd[1]: Stopping Ceph mon.smithi167 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:48:29.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon-smithi167[138922]: 2023-11-16T13:48:28.848+0000 7f75b19d4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi167 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-11-16T13:48:29.097 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:28 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon-smithi167[138922]: 2023-11-16T13:48:28.848+0000 7f75b19d4700 -1 mon.smithi167@1(peon) e3 *** Got Signal Terminated *** 2023-11-16T13:48:29.481 INFO:journalctl@ceph.mon.smithi167.smithi167.stdout:Nov 16 13:48:29 smithi167 bash[171104]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-mon-smithi167 2023-11-16T13:48:29.487 DEBUG:teuthology.orchestra.run.smithi167:> sudo pkill -f 'journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@mon.smithi167.service' 2023-11-16T13:48:29.534 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-11-16T13:48:29.534 INFO:tasks.cephadm.mon.smithi167:Stopped mon.smithi167 2023-11-16T13:48:29.534 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-11-16T13:48:29.534 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.0 2023-11-16T13:48:29.984 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:48:29 smithi151 systemd[1]: Stopping Ceph osd.0 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:48:29.984 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:48:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-0[162938]: 2023-11-16T13:48:29.741+0000 7f889465e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-11-16T13:48:29.984 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:48:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-0[162938]: 2023-11-16T13:48:29.741+0000 7f889465e700 -1 osd.0 109 *** Got signal Terminated *** 2023-11-16T13:48:29.984 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:48:29 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-0[162938]: 2023-11-16T13:48:29.741+0000 7f889465e700 -1 osd.0 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:48:35.484 INFO:journalctl@ceph.osd.0.smithi151.stdout:Nov 16 13:48:35 smithi151 bash[213454]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-0 2023-11-16T13:48:36.390 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.0.service' 2023-11-16T13:48:36.430 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-11-16T13:48:36.431 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-11-16T13:48:36.431 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-11-16T13:48:36.431 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.1 2023-11-16T13:48:36.681 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:48:36 smithi151 systemd[1]: Stopping Ceph osd.1 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:48:36.984 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:48:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-1[166153]: 2023-11-16T13:48:36.677+0000 7f3378acd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-11-16T13:48:36.984 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:48:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-1[166153]: 2023-11-16T13:48:36.677+0000 7f3378acd700 -1 osd.1 109 *** Got signal Terminated *** 2023-11-16T13:48:36.984 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:48:36 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-1[166153]: 2023-11-16T13:48:36.677+0000 7f3378acd700 -1 osd.1 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:48:42.484 INFO:journalctl@ceph.osd.1.smithi151.stdout:Nov 16 13:48:42 smithi151 bash[213713]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-1 2023-11-16T13:48:43.447 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.1.service' 2023-11-16T13:48:43.487 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-11-16T13:48:43.487 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-11-16T13:48:43.487 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-11-16T13:48:43.487 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.2 2023-11-16T13:48:43.734 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:48:43 smithi151 systemd[1]: Stopping Ceph osd.2 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:48:44.234 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:48:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-2[169487]: 2023-11-16T13:48:43.731+0000 7f5575780700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-11-16T13:48:44.234 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:48:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-2[169487]: 2023-11-16T13:48:43.731+0000 7f5575780700 -1 osd.2 109 *** Got signal Terminated *** 2023-11-16T13:48:44.234 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:48:43 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-2[169487]: 2023-11-16T13:48:43.731+0000 7f5575780700 -1 osd.2 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:48:49.734 INFO:journalctl@ceph.osd.2.smithi151.stdout:Nov 16 13:48:49 smithi151 bash[214009]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-2 2023-11-16T13:48:50.572 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.2.service' 2023-11-16T13:48:50.613 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-11-16T13:48:50.613 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-11-16T13:48:50.613 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-11-16T13:48:50.614 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.3 2023-11-16T13:48:50.852 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:48:50 smithi151 systemd[1]: Stopping Ceph osd.3 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:48:51.234 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:48:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-3[172531]: 2023-11-16T13:48:50.876+0000 7f03507df700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-11-16T13:48:51.234 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:48:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-3[172531]: 2023-11-16T13:48:50.876+0000 7f03507df700 -1 osd.3 109 *** Got signal Terminated *** 2023-11-16T13:48:51.234 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:48:50 smithi151 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-3[172531]: 2023-11-16T13:48:50.876+0000 7f03507df700 -1 osd.3 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:48:56.346 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:48:55 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:48:55.869+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:48:56.984 INFO:journalctl@ceph.osd.3.smithi151.stdout:Nov 16 13:48:56 smithi151 bash[214272]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-3 2023-11-16T13:48:57.096 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:48:56 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:48:56.841+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:48:57.597 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:48:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-4[144822]: 2023-11-16T13:48:57.332+0000 7f1bc55f2700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.967084+0000 front 2023-11-16T13:48:31.967288+0000 (oldest deadline 2023-11-16T13:48:56.667131+0000) 2023-11-16T13:48:57.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:48:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:48:57.437+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:48:57.770 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.3.service' 2023-11-16T13:48:57.811 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-11-16T13:48:57.811 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-11-16T13:48:57.811 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-11-16T13:48:57.811 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.4 2023-11-16T13:48:58.097 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:48:57 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:48:57.828+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:48:58.097 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:48:57 smithi167 systemd[1]: Stopping Ceph osd.4 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:48:58.097 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:48:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-4[144822]: 2023-11-16T13:48:58.037+0000 7f1bc8ff4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-11-16T13:48:58.097 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:48:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-4[144822]: 2023-11-16T13:48:58.037+0000 7f1bc8ff4700 -1 osd.4 109 *** Got signal Terminated *** 2023-11-16T13:48:58.097 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:48:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-4[144822]: 2023-11-16T13:48:58.037+0000 7f1bc8ff4700 -1 osd.4 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:48:58.596 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:48:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-4[144822]: 2023-11-16T13:48:58.288+0000 7f1bc55f2700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.967084+0000 front 2023-11-16T13:48:31.967288+0000 (oldest deadline 2023-11-16T13:48:56.667131+0000) 2023-11-16T13:48:58.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:48:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:48:58.415+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:48:59.096 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:48:58 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:48:58.830+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:48:59.596 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:48:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:48:59.401+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:48:59.597 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:48:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-4[144822]: 2023-11-16T13:48:59.327+0000 7f1bc55f2700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.967084+0000 front 2023-11-16T13:48:31.967288+0000 (oldest deadline 2023-11-16T13:48:56.667131+0000) 2023-11-16T13:49:00.159 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:48:59 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:48:59.865+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:49:00.596 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:00.405+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:00.597 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:49:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-4[144822]: 2023-11-16T13:49:00.368+0000 7f1bc55f2700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.967084+0000 front 2023-11-16T13:48:31.967288+0000 (oldest deadline 2023-11-16T13:48:56.667131+0000) 2023-11-16T13:49:00.597 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:00.157+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:01.145 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:00 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:00.893+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:49:01.596 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:01.371+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:01.597 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:01.143+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:01.597 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:49:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-4[144822]: 2023-11-16T13:49:01.384+0000 7f1bc55f2700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.967084+0000 front 2023-11-16T13:48:31.967288+0000 (oldest deadline 2023-11-16T13:48:56.667131+0000) 2023-11-16T13:49:02.346 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:01 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:01.904+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:49:02.347 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:02.147+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:02.347 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:02.147+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:02.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:02.407+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:02.847 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:49:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-4[144822]: 2023-11-16T13:49:02.381+0000 7f1bc55f2700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.967084+0000 front 2023-11-16T13:48:31.967288+0000 (oldest deadline 2023-11-16T13:48:56.667131+0000) 2023-11-16T13:49:03.346 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:02.950+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:49:03.347 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:02 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:02.950+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:35.818389+0000 front 2023-11-16T13:48:35.818426+0000 (oldest deadline 2023-11-16T13:49:02.218723+0000) 2023-11-16T13:49:03.347 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:03.177+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:03.347 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:03.177+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:03.717 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:03.399+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:04.096 INFO:journalctl@ceph.osd.4.smithi167.stdout:Nov 16 13:49:03 smithi167 bash[171333]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-4 2023-11-16T13:49:04.097 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:03.945+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:49:04.097 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:03 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:03.945+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:35.818389+0000 front 2023-11-16T13:48:35.818426+0000 (oldest deadline 2023-11-16T13:49:02.218723+0000) 2023-11-16T13:49:04.572 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:04.198+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:04.572 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:04.198+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:04.572 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:04.428+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:04.890 DEBUG:teuthology.orchestra.run.smithi167:> sudo pkill -f 'journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.4.service' 2023-11-16T13:49:04.931 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-11-16T13:49:04.932 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-11-16T13:49:04.932 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-11-16T13:49:04.932 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.5 2023-11-16T13:49:05.182 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:04.915+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:49:05.182 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:04 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:04.915+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:35.818389+0000 front 2023-11-16T13:48:35.818426+0000 (oldest deadline 2023-11-16T13:49:02.218723+0000) 2023-11-16T13:49:05.182 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:05 smithi167 systemd[1]: Stopping Ceph osd.5 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:49:05.597 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:05.179+0000 7fec2b6b8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-11-16T13:49:05.597 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:05.179+0000 7fec2b6b8700 -1 osd.5 109 *** Got signal Terminated *** 2023-11-16T13:49:05.597 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:05.179+0000 7fec2b6b8700 -1 osd.5 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:49:05.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:05.425+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:05.597 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:05.188+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:05.597 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:05.188+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:06.208 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:05.924+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:49:06.208 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:05 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:05.924+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:35.818389+0000 front 2023-11-16T13:48:35.818426+0000 (oldest deadline 2023-11-16T13:49:02.218723+0000) 2023-11-16T13:49:06.596 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:06.390+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:06.597 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:06.206+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:06.597 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:06.206+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:07.190 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:06.888+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:49:07.190 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:06 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:06.888+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:35.818389+0000 front 2023-11-16T13:48:35.818426+0000 (oldest deadline 2023-11-16T13:49:02.218723+0000) 2023-11-16T13:49:07.596 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:07.188+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:07.596 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:07.188+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:07.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:07.398+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:07.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:07.398+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:08.226 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:07.846+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:49:08.226 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:07 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:07.846+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:35.818389+0000 front 2023-11-16T13:48:35.818426+0000 (oldest deadline 2023-11-16T13:49:02.218723+0000) 2023-11-16T13:49:08.596 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:08.223+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:08.596 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:08.223+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:08.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:08.396+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:08.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:08.396+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:09.096 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:08.839+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:49:09.096 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:08 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:08.839+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:35.818389+0000 front 2023-11-16T13:48:35.818426+0000 (oldest deadline 2023-11-16T13:49:02.218723+0000) 2023-11-16T13:49:09.596 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:09.260+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:09.596 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:09.260+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:09.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:09.440+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:09.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:09.440+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:10.096 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:09.843+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:34.118185+0000 front 2023-11-16T13:48:34.118208+0000 (oldest deadline 2023-11-16T13:48:55.817993+0000) 2023-11-16T13:49:10.096 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:09.843+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:35.818389+0000 front 2023-11-16T13:48:35.818426+0000 (oldest deadline 2023-11-16T13:49:02.218723+0000) 2023-11-16T13:49:10.097 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:09 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5[147547]: 2023-11-16T13:49:09.843+0000 7fec264b3700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:46.919665+0000 front 2023-11-16T13:48:46.919716+0000 (oldest deadline 2023-11-16T13:49:09.819601+0000) 2023-11-16T13:49:10.596 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:10.429+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:10.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:10.429+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:10.597 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:10.268+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:10.597 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:10.268+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:10.597 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:10 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:10.268+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.737273+0000 front 2023-11-16T13:48:47.737288+0000 (oldest deadline 2023-11-16T13:49:10.037313+0000) 2023-11-16T13:49:11.096 INFO:journalctl@ceph.osd.5.smithi167.stdout:Nov 16 13:49:10 smithi167 bash[171595]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-5 2023-11-16T13:49:11.478 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:11.221+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:11.478 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:11.221+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:11.478 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:11.221+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.737273+0000 front 2023-11-16T13:48:47.737288+0000 (oldest deadline 2023-11-16T13:49:10.037313+0000) 2023-11-16T13:49:11.739 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:11.475+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:11.740 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:11.475+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:11.740 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:11 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:11.475+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.036765+0000 front 2023-11-16T13:48:47.036779+0000 (oldest deadline 2023-11-16T13:49:11.136709+0000) 2023-11-16T13:49:11.949 DEBUG:teuthology.orchestra.run.smithi167:> sudo pkill -f 'journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.5.service' 2023-11-16T13:49:11.988 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-11-16T13:49:11.989 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-11-16T13:49:11.989 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-11-16T13:49:11.989 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.6 2023-11-16T13:49:12.347 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:12 smithi167 systemd[1]: Stopping Ceph osd.6 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:49:12.347 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:12.181+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:12.347 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:12.181+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:12.347 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:12.181+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.737273+0000 front 2023-11-16T13:48:47.737288+0000 (oldest deadline 2023-11-16T13:49:10.037313+0000) 2023-11-16T13:49:12.347 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:12.227+0000 7f02d0082700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-11-16T13:49:12.347 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:12.227+0000 7f02d0082700 -1 osd.6 109 *** Got signal Terminated *** 2023-11-16T13:49:12.347 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:12.227+0000 7f02d0082700 -1 osd.6 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:49:12.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:12.451+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:12.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:12.451+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:12.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:12 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:12.451+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.036765+0000 front 2023-11-16T13:48:47.036779+0000 (oldest deadline 2023-11-16T13:49:11.136709+0000) 2023-11-16T13:49:13.442 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:13.160+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:13.442 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:13.160+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:13.442 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:13.160+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.737273+0000 front 2023-11-16T13:48:47.737288+0000 (oldest deadline 2023-11-16T13:49:10.037313+0000) 2023-11-16T13:49:13.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:13.439+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:13.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:13.439+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:13.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:13 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:13.439+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.036765+0000 front 2023-11-16T13:48:47.036779+0000 (oldest deadline 2023-11-16T13:49:11.136709+0000) 2023-11-16T13:49:14.478 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:14.146+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:14.479 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:14.146+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:14.479 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:14.146+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.737273+0000 front 2023-11-16T13:48:47.737288+0000 (oldest deadline 2023-11-16T13:49:10.037313+0000) 2023-11-16T13:49:14.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:14.476+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:14.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:14.476+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:14.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:14 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:14.476+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.036765+0000 front 2023-11-16T13:48:47.036779+0000 (oldest deadline 2023-11-16T13:49:11.136709+0000) 2023-11-16T13:49:15.447 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:15.166+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:15.447 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:15.166+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:15.447 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:15.166+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.737273+0000 front 2023-11-16T13:48:47.737288+0000 (oldest deadline 2023-11-16T13:49:10.037313+0000) 2023-11-16T13:49:15.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:15.443+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:15.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:15.443+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:15.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:15 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:15.443+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.036765+0000 front 2023-11-16T13:48:47.036779+0000 (oldest deadline 2023-11-16T13:49:11.136709+0000) 2023-11-16T13:49:16.462 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:16.156+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:16.463 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:16.156+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:16.463 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:16.156+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.737273+0000 front 2023-11-16T13:48:47.737288+0000 (oldest deadline 2023-11-16T13:49:10.037313+0000) 2023-11-16T13:49:16.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:16.460+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:16.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:16.460+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:16.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:16 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:16.460+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.036765+0000 front 2023-11-16T13:48:47.036779+0000 (oldest deadline 2023-11-16T13:49:11.136709+0000) 2023-11-16T13:49:17.483 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:17.204+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:33.934598+0000 front 2023-11-16T13:48:33.934384+0000 (oldest deadline 2023-11-16T13:48:59.234371+0000) 2023-11-16T13:49:17.484 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:17.204+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.535418+0000 front 2023-11-16T13:48:41.535446+0000 (oldest deadline 2023-11-16T13:49:02.035270+0000) 2023-11-16T13:49:17.484 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6[150243]: 2023-11-16T13:49:17.204+0000 7f02cde83700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.737273+0000 front 2023-11-16T13:48:47.737288+0000 (oldest deadline 2023-11-16T13:49:10.037313+0000) 2023-11-16T13:49:17.797 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:17.481+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:17.797 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:17.481+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:17.797 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:17 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:17.481+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.036765+0000 front 2023-11-16T13:48:47.036779+0000 (oldest deadline 2023-11-16T13:49:11.136709+0000) 2023-11-16T13:49:18.096 INFO:journalctl@ceph.osd.6.smithi167.stdout:Nov 16 13:49:17 smithi167 bash[171857]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-6 2023-11-16T13:49:18.596 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:18.511+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:18.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:18.511+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:18.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:18.511+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.036765+0000 front 2023-11-16T13:48:47.036779+0000 (oldest deadline 2023-11-16T13:49:11.136709+0000) 2023-11-16T13:49:18.597 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:18 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:18.511+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6830 osd.3 since back 2023-11-16T13:48:55.038397+0000 front 2023-11-16T13:48:55.038340+0000 (oldest deadline 2023-11-16T13:49:17.938271+0000) 2023-11-16T13:49:18.989 DEBUG:teuthology.orchestra.run.smithi167:> sudo pkill -f 'journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.6.service' 2023-11-16T13:49:19.028 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-11-16T13:49:19.028 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-11-16T13:49:19.029 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-11-16T13:49:19.029 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.7 2023-11-16T13:49:19.350 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:19 smithi167 systemd[1]: Stopping Ceph osd.7 for 11fdb452-8484-11ee-8dbe-212e2dc638e7... 2023-11-16T13:49:19.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:19.367+0000 7fea62867700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-11-16T13:49:19.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:19.367+0000 7fea62867700 -1 osd.7 109 *** Got signal Terminated *** 2023-11-16T13:49:19.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:19.367+0000 7fea62867700 -1 osd.7 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-11-16T13:49:19.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:19.527+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:19.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:19.527+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:19.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:19.527+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.036765+0000 front 2023-11-16T13:48:47.036779+0000 (oldest deadline 2023-11-16T13:49:11.136709+0000) 2023-11-16T13:49:19.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:19 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:19.527+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6830 osd.3 since back 2023-11-16T13:48:55.038397+0000 front 2023-11-16T13:48:55.038340+0000 (oldest deadline 2023-11-16T13:49:17.938271+0000) 2023-11-16T13:49:20.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:20 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:20.570+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:20.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:20 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:20.570+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:20.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:20 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:20.570+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.036765+0000 front 2023-11-16T13:48:47.036779+0000 (oldest deadline 2023-11-16T13:49:11.136709+0000) 2023-11-16T13:49:20.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:20 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:20.570+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6830 osd.3 since back 2023-11-16T13:48:55.038397+0000 front 2023-11-16T13:48:55.038340+0000 (oldest deadline 2023-11-16T13:49:17.938271+0000) 2023-11-16T13:49:21.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:21.546+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:21.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:21.546+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:21.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:21.546+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.036765+0000 front 2023-11-16T13:48:47.036779+0000 (oldest deadline 2023-11-16T13:49:11.136709+0000) 2023-11-16T13:49:21.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:21 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:21.546+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6830 osd.3 since back 2023-11-16T13:48:55.038397+0000 front 2023-11-16T13:48:55.038340+0000 (oldest deadline 2023-11-16T13:49:17.938271+0000) 2023-11-16T13:49:22.846 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:22.508+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:22.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:22.508+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:22.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:22.508+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.036765+0000 front 2023-11-16T13:48:47.036779+0000 (oldest deadline 2023-11-16T13:49:11.136709+0000) 2023-11-16T13:49:22.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:22 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:22.508+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6830 osd.3 since back 2023-11-16T13:48:55.038397+0000 front 2023-11-16T13:48:55.038340+0000 (oldest deadline 2023-11-16T13:49:17.938271+0000) 2023-11-16T13:49:23.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:23.556+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6806 osd.0 since back 2023-11-16T13:48:31.135391+0000 front 2023-11-16T13:48:31.135453+0000 (oldest deadline 2023-11-16T13:48:57.035326+0000) 2023-11-16T13:49:23.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:23.556+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6814 osd.1 since back 2023-11-16T13:48:41.136313+0000 front 2023-11-16T13:48:41.136378+0000 (oldest deadline 2023-11-16T13:49:07.036321+0000) 2023-11-16T13:49:23.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:23.556+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6822 osd.2 since back 2023-11-16T13:48:47.036765+0000 front 2023-11-16T13:48:47.036779+0000 (oldest deadline 2023-11-16T13:49:11.136709+0000) 2023-11-16T13:49:23.847 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:23 smithi167 ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7[152905]: 2023-11-16T13:49:23.556+0000 7fea5fe67700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.151:6830 osd.3 since back 2023-11-16T13:48:55.038397+0000 front 2023-11-16T13:48:55.038340+0000 (oldest deadline 2023-11-16T13:49:17.938271+0000) 2023-11-16T13:49:25.347 INFO:journalctl@ceph.osd.7.smithi167.stdout:Nov 16 13:49:24 smithi167 bash[172159]: ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7-osd-7 2023-11-16T13:49:26.129 DEBUG:teuthology.orchestra.run.smithi167:> sudo pkill -f 'journalctl -f -n 0 -u ceph-11fdb452-8484-11ee-8dbe-212e2dc638e7@osd.7.service' 2023-11-16T13:49:26.168 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-11-16T13:49:26.169 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-11-16T13:49:26.169 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 --force --keep-logs 2023-11-16T13:52:20.901 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 --force --keep-logs 2023-11-16T13:52:36.514 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-11-16T13:52:36.554 DEBUG:teuthology.orchestra.run.smithi167:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-11-16T13:52:36.589 INFO:tasks.cephadm:Archiving crash dumps... 2023-11-16T13:52:36.591 DEBUG:teuthology.misc:Transferring archived files from smithi151:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/crash to /home/teuthworker/archive/gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi/7460760/remote/smithi151/crash 2023-11-16T13:52:36.592 DEBUG:teuthology.orchestra.run.smithi151:> sudo tar c -f - -C /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/crash -- . 2023-11-16T13:52:36.637 INFO:teuthology.orchestra.run.smithi151.stderr:tar: /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/crash: Cannot open: No such file or directory 2023-11-16T13:52:36.638 INFO:teuthology.orchestra.run.smithi151.stderr:tar: Error is not recoverable: exiting now 2023-11-16T13:52:36.640 DEBUG:teuthology.misc:Transferring archived files from smithi167:/var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/crash to /home/teuthworker/archive/gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi/7460760/remote/smithi167/crash 2023-11-16T13:52:36.641 DEBUG:teuthology.orchestra.run.smithi167:> sudo tar c -f - -C /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/crash -- . 2023-11-16T13:52:36.697 INFO:teuthology.orchestra.run.smithi167.stderr:tar: /var/lib/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/crash: Cannot open: No such file or directory 2023-11-16T13:52:36.698 INFO:teuthology.orchestra.run.smithi167.stderr:tar: Error is not recoverable: exiting now 2023-11-16T13:52:36.699 INFO:tasks.cephadm:Checking cluster log for badness... 2023-11-16T13:52:36.700 DEBUG:teuthology.orchestra.run.smithi151:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-11-16T13:52:36.736 INFO:teuthology.orchestra.run.smithi151.stderr:grep: /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph.log: No such file or directory 2023-11-16T13:52:36.736 INFO:tasks.cephadm:Compressing logs... 2023-11-16T13:52:36.737 DEBUG:teuthology.orchestra.run.smithi151:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2023-11-16T13:52:36.783 DEBUG:teuthology.orchestra.run.smithi167:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2023-11-16T13:52:36.808 INFO:teuthology.orchestra.run.smithi151.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2023-11-16T13:52:36.808 INFO:teuthology.orchestra.run.smithi151.stderr:: No such file or directory 2023-11-16T13:52:36.809 INFO:teuthology.orchestra.run.smithi151.stderr:gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mgr.smithi151.fcujra.log 2023-11-16T13:52:36.809 INFO:teuthology.orchestra.run.smithi151.stderr:gzip /var/log/ceph/cephadm.log: -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-volume.log 2023-11-16T13:52:36.809 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mgr.smithi151.fcujra.log: gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.2.log 2023-11-16T13:52:36.811 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.3.log 2023-11-16T13:52:36.811 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.0.log 2023-11-16T13:52:36.811 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mds.foofs.smithi151.mdkcfw.log 2023-11-16T13:52:36.812 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.0.log: /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mds.foofs.smithi151.mdkcfw.log: 86.2% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mds.foofs.smithi151.mdkcfw.log.gz 2023-11-16T13:52:36.812 INFO:teuthology.orchestra.run.smithi151.stderr:gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mon.smithi151.log 2023-11-16T13:52:36.812 INFO:teuthology.orchestra.run.smithi167.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2023-11-16T13:52:36.812 INFO:teuthology.orchestra.run.smithi167.stderr:: No such file or directory 2023-11-16T13:52:36.812 INFO:teuthology.orchestra.run.smithi167.stderr:gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.6.log 2023-11-16T13:52:36.812 INFO:teuthology.orchestra.run.smithi167.stderr:gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-volume.log 2023-11-16T13:52:36.812 INFO:teuthology.orchestra.run.smithi167.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mgr.smithi167.xxnzsz.log 2023-11-16T13:52:36.813 INFO:teuthology.orchestra.run.smithi167.stderr:/var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.7.log 2023-11-16T13:52:36.813 INFO:teuthology.orchestra.run.smithi167.stderr:gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mon.smithi167.log 2023-11-16T13:52:36.813 INFO:teuthology.orchestra.run.smithi167.stderr:/var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mgr.smithi167.xxnzsz.log: /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.4.log 2023-11-16T13:52:36.816 INFO:teuthology.orchestra.run.smithi167.stderr:/var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mon.smithi167.log: /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.4.log: 89.3% -- replaced with /var/log/ceph/cephadm.log.gz 2023-11-16T13:52:36.816 INFO:teuthology.orchestra.run.smithi167.stderr:gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.5.log 2023-11-16T13:52:36.817 INFO:teuthology.orchestra.run.smithi167.stderr:/var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mds.foofs.smithi167.irznoa.log 2023-11-16T13:52:36.817 INFO:teuthology.orchestra.run.smithi167.stderr:/var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mds.foofs.smithi167.irznoa.log: 80.1% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mds.foofs.smithi167.irznoa.log.gz 2023-11-16T13:52:36.818 INFO:teuthology.orchestra.run.smithi167.stderr: 90.8% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-volume.log.gz 2023-11-16T13:52:36.820 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mon.smithi151.log: 90.1% -- replaced with /var/log/ceph/cephadm.log.gz 2023-11-16T13:52:36.821 INFO:teuthology.orchestra.run.smithi151.stderr: 91.0% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-volume.log.gz 2023-11-16T13:52:36.821 INFO:teuthology.orchestra.run.smithi151.stderr:gzip -5 --verbose -- /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.1.log 2023-11-16T13:52:36.882 INFO:teuthology.orchestra.run.smithi167.stderr: 89.4% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mgr.smithi167.xxnzsz.log.gz 2023-11-16T13:52:37.040 INFO:teuthology.orchestra.run.smithi167.stderr: 92.7% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mon.smithi167.log.gz 2023-11-16T13:52:37.182 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.1.log: 90.5% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mgr.smithi151.fcujra.log.gz 2023-11-16T13:52:37.769 INFO:teuthology.orchestra.run.smithi151.stderr: 91.6% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-mon.smithi151.log.gz 2023-11-16T13:52:39.856 INFO:teuthology.orchestra.run.smithi151.stderr: 94.3% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.2.log.gz 2023-11-16T13:52:41.580 INFO:teuthology.orchestra.run.smithi167.stderr: 94.2% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.6.log.gz 2023-11-16T13:52:42.005 INFO:teuthology.orchestra.run.smithi167.stderr: 94.0% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.4.log.gz 2023-11-16T13:52:42.187 INFO:teuthology.orchestra.run.smithi151.stderr: 94.1% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.0.log.gz 2023-11-16T13:52:42.756 INFO:teuthology.orchestra.run.smithi151.stderr: 94.2% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.1.log.gz 2023-11-16T13:52:42.902 INFO:teuthology.orchestra.run.smithi151.stderr: 94.0% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.3.log.gz 2023-11-16T13:52:42.904 INFO:teuthology.orchestra.run.smithi151.stderr: 2023-11-16T13:52:42.904 INFO:teuthology.orchestra.run.smithi151.stderr:real 0m6.107s 2023-11-16T13:52:42.904 INFO:teuthology.orchestra.run.smithi151.stderr:user 0m20.933s 2023-11-16T13:52:42.904 INFO:teuthology.orchestra.run.smithi151.stderr:sys 0m0.863s 2023-11-16T13:52:42.922 INFO:teuthology.orchestra.run.smithi167.stderr: 94.1% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.5.log.gz 2023-11-16T13:52:45.127 INFO:teuthology.orchestra.run.smithi167.stderr: 94.1% -- replaced with /var/log/ceph/11fdb452-8484-11ee-8dbe-212e2dc638e7/ceph-osd.7.log.gz 2023-11-16T13:52:45.127 INFO:teuthology.orchestra.run.smithi167.stderr: 2023-11-16T13:52:45.128 INFO:teuthology.orchestra.run.smithi167.stderr:real 0m8.326s 2023-11-16T13:52:45.128 INFO:teuthology.orchestra.run.smithi167.stderr:user 0m23.702s 2023-11-16T13:52:45.128 INFO:teuthology.orchestra.run.smithi167.stderr:sys 0m0.985s 2023-11-16T13:52:45.128 INFO:tasks.cephadm:Archiving logs... 2023-11-16T13:52:45.128 DEBUG:teuthology.misc:Transferring archived files from smithi151:/var/log/ceph to /home/teuthworker/archive/gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi/7460760/remote/smithi151/log 2023-11-16T13:52:45.129 DEBUG:teuthology.orchestra.run.smithi151:> sudo tar c -f - -C /var/log/ceph -- . 2023-11-16T13:52:48.215 DEBUG:teuthology.misc:Transferring archived files from smithi167:/var/log/ceph to /home/teuthworker/archive/gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi/7460760/remote/smithi167/log 2023-11-16T13:52:48.216 DEBUG:teuthology.orchestra.run.smithi167:> sudo tar c -f - -C /var/log/ceph -- . 2023-11-16T13:52:51.870 INFO:tasks.cephadm:Removing cluster... 2023-11-16T13:52:51.871 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 --force 2023-11-16T13:52:52.216 DEBUG:teuthology.orchestra.run.smithi167:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 11fdb452-8484-11ee-8dbe-212e2dc638e7 --force 2023-11-16T13:52:52.565 INFO:tasks.cephadm:Removing cephadm ... 2023-11-16T13:52:52.565 DEBUG:teuthology.orchestra.run.smithi151:> rm -rf /home/ubuntu/cephtest/cephadm 2023-11-16T13:52:52.584 DEBUG:teuthology.orchestra.run.smithi167:> rm -rf /home/ubuntu/cephtest/cephadm 2023-11-16T13:52:52.603 INFO:tasks.cephadm:Teardown complete 2023-11-16T13:52:52.603 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-11-16T13:52:52.616 INFO:teuthology.task.clock:Checking final clock skew... 2023-11-16T13:52:52.616 DEBUG:teuthology.orchestra.run.smithi151:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-11-16T13:52:52.628 DEBUG:teuthology.orchestra.run.smithi167:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-11-16T13:52:52.642 INFO:teuthology.orchestra.run.smithi151.stderr:bash: ntpq: command not found 2023-11-16T13:52:52.650 INFO:teuthology.orchestra.run.smithi151.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-11-16T13:52:52.651 INFO:teuthology.orchestra.run.smithi151.stdout:=============================================================================== 2023-11-16T13:52:52.651 INFO:teuthology.orchestra.run.smithi151.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 48 +763us[ +763us] +/- 73ms 2023-11-16T13:52:52.651 INFO:teuthology.orchestra.run.smithi151.stdout:^* hv02.front.sepia.ceph.com 3 7 377 111 -628us[ -612us] +/- 71ms 2023-11-16T13:52:52.651 INFO:teuthology.orchestra.run.smithi151.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 43 +217us[ +217us] +/- 70ms 2023-11-16T13:52:52.651 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2023-11-16T13:52:52.659 INFO:teuthology.orchestra.run.smithi167.stderr:bash: ntpq: command not found 2023-11-16T13:52:52.664 INFO:teuthology.orchestra.run.smithi167.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-11-16T13:52:52.664 INFO:teuthology.orchestra.run.smithi167.stdout:=============================================================================== 2023-11-16T13:52:52.665 INFO:teuthology.orchestra.run.smithi167.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 45 +674us[ +674us] +/- 73ms 2023-11-16T13:52:52.665 INFO:teuthology.orchestra.run.smithi167.stdout:^* hv02.front.sepia.ceph.com 3 6 377 47 -752us[ -704us] +/- 72ms 2023-11-16T13:52:52.665 INFO:teuthology.orchestra.run.smithi167.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 48 +119us[ +167us] +/- 70ms 2023-11-16T13:52:52.665 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2023-11-16T13:52:52.665 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-11-16T13:52:52.678 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-11-16T13:52:52.679 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-11-16T13:52:52.726 DEBUG:teuthology.orchestra.run.smithi151:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-11-16T13:52:52.823 DEBUG:teuthology.orchestra.run.smithi167:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-11-16T13:52:52.915 DEBUG:teuthology.orchestra.run.smithi151:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-11-16T13:52:52.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-16T13:52:52.948 DEBUG:teuthology.orchestra.run.smithi167:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-11-16T13:52:52.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-16T13:52:52.980 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-11-16T13:52:52.992 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-11-16T13:52:53.018 INFO:teuthology.task.internal:Duration was 1942.617789 seconds 2023-11-16T13:52:53.019 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-11-16T13:52:53.068 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-11-16T13:52:53.068 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-11-16T13:52:53.070 DEBUG:teuthology.orchestra.run.smithi167:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-11-16T13:52:53.113 INFO:teuthology.orchestra.run.smithi151.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-11-16T13:52:53.116 INFO:teuthology.orchestra.run.smithi167.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-11-16T13:52:53.608 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-11-16T13:52:53.609 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi151.front.sepia.ceph.com 2023-11-16T13:52:53.609 DEBUG:teuthology.orchestra.run.smithi151:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-11-16T13:52:53.632 INFO:teuthology.orchestra.run.smithi151.stdout:2023-11-16T13:45:28.923776+00:00 smithi151 kernel: rcu: INFO: rcu_sched self-detected stall on CPU 2023-11-16T13:52:53.632 ERROR:teuthology.task.internal.syslog:Error in syslog on ubuntu@smithi151.front.sepia.ceph.com: 2023-11-16T13:45:28.923776+00:00 smithi151 kernel: rcu: INFO: rcu_sched self-detected stall on CPU 2023-11-16T13:52:53.632 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi167.front.sepia.ceph.com 2023-11-16T13:52:53.633 DEBUG:teuthology.orchestra.run.smithi167:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-11-16T13:52:53.658 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-11-16T13:52:53.659 DEBUG:teuthology.orchestra.run.smithi151:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-11-16T13:52:53.674 DEBUG:teuthology.orchestra.run.smithi167:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-11-16T13:52:53.755 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-11-16T13:52:53.755 DEBUG:teuthology.orchestra.run.smithi151:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-11-16T13:52:53.757 DEBUG:teuthology.orchestra.run.smithi167:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-11-16T13:52:53.963 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-11-16T13:52:53.975 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-11-16T13:52:53.975 DEBUG:teuthology.orchestra.run.smithi151:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-11-16T13:52:54.003 DEBUG:teuthology.orchestra.run.smithi167:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-11-16T13:52:54.030 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-11-16T13:52:54.052 DEBUG:teuthology.orchestra.run.smithi151:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-11-16T13:52:54.054 DEBUG:teuthology.orchestra.run.smithi167:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-11-16T13:52:54.078 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern = core 2023-11-16T13:52:54.097 INFO:teuthology.orchestra.run.smithi167.stdout:kernel.core_pattern = core 2023-11-16T13:52:54.121 DEBUG:teuthology.orchestra.run.smithi151:> test -e /home/ubuntu/cephtest/archive/coredump 2023-11-16T13:52:54.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-16T13:52:54.156 DEBUG:teuthology.orchestra.run.smithi167:> test -e /home/ubuntu/cephtest/archive/coredump 2023-11-16T13:52:54.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-16T13:52:54.179 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-11-16T13:52:54.191 INFO:teuthology.task.internal:Transferring archived files... 2023-11-16T13:52:54.192 DEBUG:teuthology.misc:Transferring archived files from smithi151:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi/7460760/remote/smithi151 2023-11-16T13:52:54.192 DEBUG:teuthology.orchestra.run.smithi151:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-11-16T13:52:54.245 DEBUG:teuthology.misc:Transferring archived files from smithi167:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi/7460760/remote/smithi167 2023-11-16T13:52:54.245 DEBUG:teuthology.orchestra.run.smithi167:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-11-16T13:52:54.287 INFO:teuthology.task.internal:Removing archive directory... 2023-11-16T13:52:54.287 DEBUG:teuthology.orchestra.run.smithi151:> rm -rf -- /home/ubuntu/cephtest/archive 2023-11-16T13:52:54.290 DEBUG:teuthology.orchestra.run.smithi167:> rm -rf -- /home/ubuntu/cephtest/archive 2023-11-16T13:52:54.337 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-11-16T13:52:54.349 INFO:teuthology.task.internal:Not uploading archives. 2023-11-16T13:52:54.349 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-11-16T13:52:54.388 INFO:teuthology.task.internal:Tidying up after the test... 2023-11-16T13:52:54.388 DEBUG:teuthology.orchestra.run.smithi151:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-11-16T13:52:54.390 DEBUG:teuthology.orchestra.run.smithi167:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-11-16T13:52:54.405 INFO:teuthology.orchestra.run.smithi151.stdout: 525841 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Nov 16 13:52 /home/ubuntu/cephtest 2023-11-16T13:52:54.411 INFO:teuthology.orchestra.run.smithi167.stdout: 525357 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Nov 16 13:52 /home/ubuntu/cephtest 2023-11-16T13:52:54.413 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-11-16T13:52:54.438 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-11-16T13:52:54.539 INFO:teuthology.nuke:Checking targets against current locks 2023-11-16T13:52:54.570 DEBUG:teuthology.nuke:shortname: smithi151 2023-11-16T13:52:54.570 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-11-16T13:52:54.596 DEBUG:teuthology.nuke:shortname: smithi167 2023-11-16T13:52:54.596 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-11-16T13:52:54.603 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi151.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi/7460760', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-11-16 13:11:54.539134', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOOEwtsar9VCyDQay7IPHqvHGH7/5fuPhSSq4FZmLsqmt1GnPFD6ECu0G16IhUHTSZNJ+9KbkoHhryOOSTUdjXA='} 2023-11-16T13:52:54.624 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi167.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2023-11-16_12:07:54-orch:cephadm-wip-63555-quincy-distro-default-smithi/7460760', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-11-16 13:11:54.540568', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGr1oPEINZRdFr3f9EY1PZNMUW1ni/iB7dWdEDYh9HGQQmKKAde/5gS6jvaLsVu0Z5kMFfiAMIPgmWunjne15I8='} 2023-11-16T13:52:54.659 INFO:teuthology.orchestra.console.smithi151:Power off 2023-11-16T13:52:54.660 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-11-16T13:52:54.688 DEBUG:teuthology.orchestra.console.smithi151:power off output: Chassis Power Control: Down/Off 2023-11-16T13:52:54.689 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-11-16T13:52:54.709 INFO:teuthology.orchestra.console.smithi167:Power off 2023-11-16T13:52:54.710 DEBUG:teuthology.orchestra.console.smithi167:pexpect command: ipmitool -H smithi167.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-11-16T13:52:54.720 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is on 2023-11-16T13:52:54.735 DEBUG:teuthology.orchestra.console.smithi167:power off output: Chassis Power Control: Down/Off 2023-11-16T13:52:54.735 DEBUG:teuthology.orchestra.console.smithi167:pexpect command: ipmitool -H smithi167.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-11-16T13:52:54.761 DEBUG:teuthology.orchestra.console.smithi167:check power output: Chassis Power is on 2023-11-16T13:52:58.721 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-11-16T13:52:58.762 DEBUG:teuthology.orchestra.console.smithi167:pexpect command: ipmitool -H smithi167.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-11-16T13:52:58.837 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is on 2023-11-16T13:52:58.877 DEBUG:teuthology.orchestra.console.smithi167:check power output: Chassis Power is on 2023-11-16T13:53:02.842 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-11-16T13:53:02.988 DEBUG:teuthology.orchestra.console.smithi167:pexpect command: ipmitool -H smithi167.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-11-16T13:53:03.090 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is on 2023-11-16T13:53:03.115 DEBUG:teuthology.orchestra.console.smithi167:check power output: Chassis Power is off 2023-11-16T13:53:03.215 INFO:teuthology.orchestra.console.smithi167:Power off completed 2023-11-16T13:53:07.094 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-11-16T13:53:07.204 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is off 2023-11-16T13:53:07.304 INFO:teuthology.orchestra.console.smithi151:Power off completed 2023-11-16T13:53:07.408 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_8.stream_container_tools 1-bootstrap/16.2.5 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1942.6177892684937 failure_reason: '''2023-11-16T13:45:28.923776+00:00 smithi151 kernel: rcu: INFO: rcu_sched self-detected stall on CPU '' in syslog' owner: scheduled_gabrioux@teuthology status: fail success: false 2023-11-16T13:53:07.409 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-11-16T13:53:07.483 INFO:teuthology.run:FAIL