2023-10-01T14:42:00.837 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-01T14:42:00.837 DEBUG:teuthology.run:Teuthology command: teuthology --description rbd:nvmeof/{0-single-container-host base/install cluster/{fixed-3 openstack} workloads/cephadm_nvmeof} --archive /home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615 --verbose --owner scheduled_bdavidov@teuthology --name bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi -- /home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615/orig.config.yaml 2023-10-01T14:42:00.878 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-01T14:42:00.953 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615 branch: wip-barakda-cephadm-nvmeof description: rbd:nvmeof/{0-single-container-host base/install cluster/{fixed-3 openstack} workloads/cephadm_nvmeof} email: null first_in_suite: false job_id: '7402615' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - machine: cpus: 1 disk: 40 ram: 8000 volumes: count: 4 size: 30 os_type: centos os_version: 8.stream overrides: admin_socket: branch: wip-barakda-cephadm-nvmeof ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: bcea8c48517ff154f392584022efa0718f51c9f2 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: bcea8c48517ff154f392584022efa0718f51c9f2 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-nvmetest-teuthology sha1: 71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5 owner: scheduled_bdavidov@teuthology priority: 1000 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - mon.a - mgr.x - osd.0 - osd.1 - client.0 - - host.b - mon.b - osd.2 - osd.3 - osd.4 - client.1 - - mon.c - osd.5 - osd.6 - osd.7 - client.2 seed: 3911 sha1: bcea8c48517ff154f392584022efa0718f51c9f2 sleep_before_teardown: 0 suite: rbd:nvmeof suite_branch: wip-nvmetest-teuthology suite_path: /home/teuthworker/src/github.com_barakda_ceph_71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5/qa suite_relpath: qa suite_repo: https://github.com/barakda/ceph suite_sha1: 71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5 targets: smithi031.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBzFxjmohYMjBlhZFHaSqP2EGsaSqCtkeQuECrh7GWlv1Hazs3K0S4MFPSfARFzwjQEVUAI9C+RJBva3kdN9j/k= smithi099.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEir+edhJYBvVeyTWMD79z8V5/Di8zfrBuQ/MOReH20F9B4nAfw2XqkY5elDs7LwzCWe4rmEVwLnKH0mppYWTDU= smithi135.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEhRzE5lRVOrZR+Y6zSbCrvxEQA40nzLw6gCAf4qkfz9R6H2+pkIEjLbJZ4gG+atSdadb2qcDmnnuKaIRokoVdI= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - sudo sed -i 's/runtime = "runc"/#runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/#runtime = "crun"/runtime = "crun"/g' /usr/share/containers/containers.conf - cephadm: null - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - ceph osd lspools - ceph osd pool create mypool - ceph osd pool application enable mypool rbd - ceph osd lspools - ceph osd pool ls detail - ceph config set mgr mgr/cephadm/container_image_nvmeof quay.io/barakda1/nvmeof:0.0.3 - ceph config get mgr mgr/cephadm/container_image_nvmeof - ceph orch apply nvmeof mypool --placement="1 $(hostname)" - ceph orch ps --refresh - ceph versions - ceph -s - ceph orch ls - rbd create mypool/myimage --size 8Gi - rbd ls mypool - 'set -ex # cat /proc/cpuinfo ' - workunit: clients: client.0: - cephadm/test_nvme_etc_hosts.sh - 'set -ex # HOSTNAME=$(hostname) IP=$(echo $(hostname -I) | cut -d '' '' -f1) echo -e "<---- exec.client.0---- HOST/IP -- $HOSTNAME/$IP ---->" ' - cephadm.wait_for_service: service: nvmeof.mypool - exec: host.a: - 'set -ex # # print nvmeof start up logs s=$(systemctl list-units | grep nvmeof.mypool | awk ''{print $1}'') PAGER=cat journalctl -u $s ' - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - ceph osd lspools - 'set -ex s=$(ceph orch ps | grep nvmeof.mypool | awk ''{print $4}'') echo $s [ $s = running ] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-26_21:57:54 tube: smithi use_shaman: true user: bdavidov verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-01T14:42:00.953 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_barakda_ceph_71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5/qa; will attempt to use it 2023-10-01T14:42:00.954 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_barakda_ceph_71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5/qa/tasks 2023-10-01T14:42:00.954 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-01T14:42:00.955 INFO:teuthology.task.internal:Checking packages... 2023-10-01T14:42:00.978 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'bcea8c48517ff154f392584022efa0718f51c9f2' 2023-10-01T14:42:00.978 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-01T14:42:00.978 INFO:teuthology.packaging:ref: None 2023-10-01T14:42:00.978 INFO:teuthology.packaging:tag: None 2023-10-01T14:42:00.978 INFO:teuthology.packaging:branch: wip-barakda-cephadm-nvmeof 2023-10-01T14:42:00.978 INFO:teuthology.packaging:sha1: bcea8c48517ff154f392584022efa0718f51c9f2 2023-10-01T14:42:00.979 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=wip-barakda-cephadm-nvmeof 2023-10-01T14:42:01.206 INFO:teuthology.task.internal:Found packages for ceph version 18.0.0-5817.gbcea8c48 2023-10-01T14:42:01.206 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-01T14:42:01.229 INFO:teuthology.task.internal:no buildpackages task found 2023-10-01T14:42:01.229 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-01T14:42:01.241 INFO:teuthology.task.internal:Saving configuration 2023-10-01T14:42:01.253 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-01T14:42:01.259 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-01T14:42:01.285 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi031.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 14:36:44.222950', 'locked_by': 'scheduled_bdavidov@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBzFxjmohYMjBlhZFHaSqP2EGsaSqCtkeQuECrh7GWlv1Hazs3K0S4MFPSfARFzwjQEVUAI9C+RJBva3kdN9j/k='} 2023-10-01T14:42:01.305 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi099.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 14:36:44.226013', 'locked_by': 'scheduled_bdavidov@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEir+edhJYBvVeyTWMD79z8V5/Di8zfrBuQ/MOReH20F9B4nAfw2XqkY5elDs7LwzCWe4rmEVwLnKH0mppYWTDU='} 2023-10-01T14:42:01.326 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi135.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 14:36:44.224882', 'locked_by': 'scheduled_bdavidov@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEhRzE5lRVOrZR+Y6zSbCrvxEQA40nzLw6gCAf4qkfz9R6H2+pkIEjLbJZ4gG+atSdadb2qcDmnnuKaIRokoVdI='} 2023-10-01T14:42:01.326 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-01T14:42:01.333 INFO:teuthology.task.internal:roles: ubuntu@smithi031.front.sepia.ceph.com - ['host.a', 'mon.a', 'mgr.x', 'osd.0', 'osd.1', 'client.0'] 2023-10-01T14:42:01.333 INFO:teuthology.task.internal:roles: ubuntu@smithi099.front.sepia.ceph.com - ['host.b', 'mon.b', 'osd.2', 'osd.3', 'osd.4', 'client.1'] 2023-10-01T14:42:01.333 INFO:teuthology.task.internal:roles: ubuntu@smithi135.front.sepia.ceph.com - ['mon.c', 'osd.5', 'osd.6', 'osd.7', 'client.2'] 2023-10-01T14:42:01.333 INFO:teuthology.run_tasks:Running task console_log... 2023-10-01T14:42:01.426 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f1764f49b80>, signals=[15]) 2023-10-01T14:42:01.427 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-01T14:42:01.432 INFO:teuthology.task.internal:Opening connections... 2023-10-01T14:42:01.432 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi031.front.sepia.ceph.com 2023-10-01T14:42:01.433 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi031.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:42:01.503 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi099.front.sepia.ceph.com 2023-10-01T14:42:01.504 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:42:01.568 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi135.front.sepia.ceph.com 2023-10-01T14:42:01.568 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:42:01.637 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-01T14:42:01.644 DEBUG:teuthology.orchestra.run.smithi031:> uname -m 2023-10-01T14:42:01.667 INFO:teuthology.orchestra.run.smithi031.stdout:x86_64 2023-10-01T14:42:01.667 DEBUG:teuthology.orchestra.run.smithi031:> cat /etc/os-release 2023-10-01T14:42:01.728 INFO:teuthology.orchestra.run.smithi031.stdout:NAME="CentOS Stream" 2023-10-01T14:42:01.728 INFO:teuthology.orchestra.run.smithi031.stdout:VERSION="8" 2023-10-01T14:42:01.729 INFO:teuthology.orchestra.run.smithi031.stdout:ID="centos" 2023-10-01T14:42:01.729 INFO:teuthology.orchestra.run.smithi031.stdout:ID_LIKE="rhel fedora" 2023-10-01T14:42:01.729 INFO:teuthology.orchestra.run.smithi031.stdout:VERSION_ID="8" 2023-10-01T14:42:01.729 INFO:teuthology.orchestra.run.smithi031.stdout:PLATFORM_ID="platform:el8" 2023-10-01T14:42:01.729 INFO:teuthology.orchestra.run.smithi031.stdout:PRETTY_NAME="CentOS Stream 8" 2023-10-01T14:42:01.729 INFO:teuthology.orchestra.run.smithi031.stdout:ANSI_COLOR="0;31" 2023-10-01T14:42:01.729 INFO:teuthology.orchestra.run.smithi031.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-10-01T14:42:01.729 INFO:teuthology.orchestra.run.smithi031.stdout:HOME_URL="https://centos.org/" 2023-10-01T14:42:01.729 INFO:teuthology.orchestra.run.smithi031.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-01T14:42:01.729 INFO:teuthology.orchestra.run.smithi031.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-01T14:42:01.729 INFO:teuthology.orchestra.run.smithi031.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-10-01T14:42:01.730 INFO:teuthology.lock.ops:Updating smithi031.front.sepia.ceph.com on lock server 2023-10-01T14:42:01.749 DEBUG:teuthology.orchestra.run.smithi099:> uname -m 2023-10-01T14:42:01.770 INFO:teuthology.orchestra.run.smithi099.stdout:x86_64 2023-10-01T14:42:01.770 DEBUG:teuthology.orchestra.run.smithi099:> cat /etc/os-release 2023-10-01T14:42:01.834 INFO:teuthology.orchestra.run.smithi099.stdout:NAME="CentOS Stream" 2023-10-01T14:42:01.834 INFO:teuthology.orchestra.run.smithi099.stdout:VERSION="8" 2023-10-01T14:42:01.834 INFO:teuthology.orchestra.run.smithi099.stdout:ID="centos" 2023-10-01T14:42:01.834 INFO:teuthology.orchestra.run.smithi099.stdout:ID_LIKE="rhel fedora" 2023-10-01T14:42:01.834 INFO:teuthology.orchestra.run.smithi099.stdout:VERSION_ID="8" 2023-10-01T14:42:01.834 INFO:teuthology.orchestra.run.smithi099.stdout:PLATFORM_ID="platform:el8" 2023-10-01T14:42:01.834 INFO:teuthology.orchestra.run.smithi099.stdout:PRETTY_NAME="CentOS Stream 8" 2023-10-01T14:42:01.834 INFO:teuthology.orchestra.run.smithi099.stdout:ANSI_COLOR="0;31" 2023-10-01T14:42:01.835 INFO:teuthology.orchestra.run.smithi099.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-10-01T14:42:01.835 INFO:teuthology.orchestra.run.smithi099.stdout:HOME_URL="https://centos.org/" 2023-10-01T14:42:01.835 INFO:teuthology.orchestra.run.smithi099.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-01T14:42:01.835 INFO:teuthology.orchestra.run.smithi099.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-01T14:42:01.835 INFO:teuthology.orchestra.run.smithi099.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-10-01T14:42:01.835 INFO:teuthology.lock.ops:Updating smithi099.front.sepia.ceph.com on lock server 2023-10-01T14:42:01.857 DEBUG:teuthology.orchestra.run.smithi135:> uname -m 2023-10-01T14:42:01.879 INFO:teuthology.orchestra.run.smithi135.stdout:x86_64 2023-10-01T14:42:01.879 DEBUG:teuthology.orchestra.run.smithi135:> cat /etc/os-release 2023-10-01T14:42:01.941 INFO:teuthology.orchestra.run.smithi135.stdout:NAME="CentOS Stream" 2023-10-01T14:42:01.941 INFO:teuthology.orchestra.run.smithi135.stdout:VERSION="8" 2023-10-01T14:42:01.941 INFO:teuthology.orchestra.run.smithi135.stdout:ID="centos" 2023-10-01T14:42:01.941 INFO:teuthology.orchestra.run.smithi135.stdout:ID_LIKE="rhel fedora" 2023-10-01T14:42:01.941 INFO:teuthology.orchestra.run.smithi135.stdout:VERSION_ID="8" 2023-10-01T14:42:01.941 INFO:teuthology.orchestra.run.smithi135.stdout:PLATFORM_ID="platform:el8" 2023-10-01T14:42:01.941 INFO:teuthology.orchestra.run.smithi135.stdout:PRETTY_NAME="CentOS Stream 8" 2023-10-01T14:42:01.941 INFO:teuthology.orchestra.run.smithi135.stdout:ANSI_COLOR="0;31" 2023-10-01T14:42:01.941 INFO:teuthology.orchestra.run.smithi135.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-10-01T14:42:01.942 INFO:teuthology.orchestra.run.smithi135.stdout:HOME_URL="https://centos.org/" 2023-10-01T14:42:01.942 INFO:teuthology.orchestra.run.smithi135.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-01T14:42:01.942 INFO:teuthology.orchestra.run.smithi135.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-01T14:42:01.942 INFO:teuthology.orchestra.run.smithi135.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-10-01T14:42:01.942 INFO:teuthology.lock.ops:Updating smithi135.front.sepia.ceph.com on lock server 2023-10-01T14:42:01.990 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-01T14:42:01.999 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-01T14:42:02.003 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-01T14:42:02.004 DEBUG:teuthology.orchestra.run.smithi031:> test '!' -e /home/ubuntu/cephtest 2023-10-01T14:42:02.006 DEBUG:teuthology.orchestra.run.smithi099:> test '!' -e /home/ubuntu/cephtest 2023-10-01T14:42:02.008 DEBUG:teuthology.orchestra.run.smithi135:> test '!' -e /home/ubuntu/cephtest 2023-10-01T14:42:02.028 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-01T14:42:02.034 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-01T14:42:02.034 DEBUG:teuthology.orchestra.run.smithi031:> test -z $(ls -A /var/lib/ceph) 2023-10-01T14:42:02.068 DEBUG:teuthology.orchestra.run.smithi099:> test -z $(ls -A /var/lib/ceph) 2023-10-01T14:42:02.071 DEBUG:teuthology.orchestra.run.smithi135:> test -z $(ls -A /var/lib/ceph) 2023-10-01T14:42:02.103 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-01T14:42:02.181 INFO:teuthology.run_tasks:Running task kernel... 2023-10-01T14:42:02.195 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-01T14:42:02.195 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}, 'host.b': {'kdb': True, 'sha1': 'distro'}, 'mon.c': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-10-01T14:42:02.196 DEBUG:teuthology.orchestra.run.smithi031:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-01T14:42:02.196 DEBUG:teuthology.orchestra.run.smithi099:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-01T14:42:02.196 DEBUG:teuthology.orchestra.run.smithi135:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-01T14:42:02.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T14:42:02.216 DEBUG:teuthology.orchestra.run.smithi099:> uname -r 2023-10-01T14:42:02.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T14:42:02.217 DEBUG:teuthology.orchestra.run.smithi031:> uname -r 2023-10-01T14:42:02.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T14:42:02.218 DEBUG:teuthology.orchestra.run.smithi135:> uname -r 2023-10-01T14:42:02.276 INFO:teuthology.orchestra.run.smithi099.stdout:4.18.0-489.el8.x86_64 2023-10-01T14:42:02.276 INFO:teuthology.task.kernel:Running kernel on smithi099: 4.18.0-489.el8.x86_64 2023-10-01T14:42:02.276 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum install -y kernel 2023-10-01T14:42:02.280 INFO:teuthology.orchestra.run.smithi031.stdout:4.18.0-489.el8.x86_64 2023-10-01T14:42:02.281 INFO:teuthology.task.kernel:Running kernel on smithi031: 4.18.0-489.el8.x86_64 2023-10-01T14:42:02.281 DEBUG:teuthology.orchestra.run.smithi031:> sudo yum install -y kernel 2023-10-01T14:42:02.283 INFO:teuthology.orchestra.run.smithi135.stdout:4.18.0-489.el8.x86_64 2023-10-01T14:42:02.283 INFO:teuthology.task.kernel:Running kernel on smithi135: 4.18.0-489.el8.x86_64 2023-10-01T14:42:02.283 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum install -y kernel 2023-10-01T14:42:07.581 INFO:teuthology.orchestra.run.smithi031.stdout:CentOS Stream 8 - AppStream 11 MB/s | 33 MB 00:02 2023-10-01T14:42:08.482 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 8 - AppStream 9.9 MB/s | 33 MB 00:03 2023-10-01T14:42:09.533 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 8 - AppStream 7.1 MB/s | 33 MB 00:04 2023-10-01T14:42:15.133 INFO:teuthology.orchestra.run.smithi031.stdout:CentOS Stream 8 - BaseOS 44 MB/s | 48 MB 00:01 2023-10-01T14:42:15.749 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 8 - BaseOS 45 MB/s | 48 MB 00:01 2023-10-01T14:42:16.641 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 8 - BaseOS 43 MB/s | 48 MB 00:01 2023-10-01T14:42:22.883 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 8 - Extras 70 kB/s | 18 kB 00:00 2023-10-01T14:42:22.889 INFO:teuthology.orchestra.run.smithi031.stdout:CentOS Stream 8 - Extras 46 kB/s | 18 kB 00:00 2023-10-01T14:42:23.409 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 8 - Extras common packages 20 kB/s | 6.9 kB 00:00 2023-10-01T14:42:23.456 INFO:teuthology.orchestra.run.smithi031.stdout:CentOS Stream 8 - Extras common packages 19 kB/s | 6.9 kB 00:00 2023-10-01T14:42:23.702 INFO:teuthology.orchestra.run.smithi135.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2023-10-01T14:42:23.758 INFO:teuthology.orchestra.run.smithi031.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2023-10-01T14:42:23.840 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 8 - Extras 45 kB/s | 18 kB 00:00 2023-10-01T14:42:24.224 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 8 - Extras common packages 33 kB/s | 6.9 kB 00:00 2023-10-01T14:42:24.501 INFO:teuthology.orchestra.run.smithi099.stdout:Copr repo for python3-asyncssh owned by ceph 27 kB/s | 3.5 kB 00:00 2023-10-01T14:42:25.658 INFO:teuthology.orchestra.run.smithi031.stdout:Extra Packages for Enterprise Linux 9.1 MB/s | 16 MB 00:01 2023-10-01T14:42:25.862 INFO:teuthology.orchestra.run.smithi135.stdout:Extra Packages for Enterprise Linux 7.9 MB/s | 16 MB 00:02 2023-10-01T14:42:26.402 INFO:teuthology.orchestra.run.smithi099.stdout:Extra Packages for Enterprise Linux 9.1 MB/s | 16 MB 00:01 2023-10-01T14:42:29.633 INFO:teuthology.orchestra.run.smithi031.stdout:lab-extras 517 kB/s | 24 kB 00:00 2023-10-01T14:42:29.662 INFO:teuthology.orchestra.run.smithi135.stdout:lab-extras 360 kB/s | 24 kB 00:00 2023-10-01T14:42:30.226 INFO:teuthology.orchestra.run.smithi099.stdout:lab-extras 270 kB/s | 24 kB 00:00 2023-10-01T14:42:33.272 INFO:teuthology.orchestra.run.smithi135.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-10-01T14:42:33.272 INFO:teuthology.orchestra.run.smithi135.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-10-01T14:42:33.348 INFO:teuthology.orchestra.run.smithi031.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-10-01T14:42:33.349 INFO:teuthology.orchestra.run.smithi031.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-10-01T14:42:33.525 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-10-01T14:42:33.526 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-10-01T14:42:33.526 INFO:teuthology.orchestra.run.smithi135.stdout: Package Architecture Version Repository Size 2023-10-01T14:42:33.526 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-10-01T14:42:33.526 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2023-10-01T14:42:33.526 INFO:teuthology.orchestra.run.smithi135.stdout: kernel x86_64 4.18.0-514.el8 baseos 10 M 2023-10-01T14:42:33.526 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2023-10-01T14:42:33.526 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-core x86_64 4.18.0-514.el8 baseos 43 M 2023-10-01T14:42:33.526 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-modules x86_64 4.18.0-514.el8 baseos 35 M 2023-10-01T14:42:33.526 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:42:33.526 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-10-01T14:42:33.527 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-10-01T14:42:33.527 INFO:teuthology.orchestra.run.smithi135.stdout:Install 3 Packages 2023-10-01T14:42:33.527 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:42:33.527 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 88 M 2023-10-01T14:42:33.527 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 96 M 2023-10-01T14:42:33.527 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-10-01T14:42:33.607 INFO:teuthology.orchestra.run.smithi031.stdout:Dependencies resolved. 2023-10-01T14:42:33.608 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2023-10-01T14:42:33.608 INFO:teuthology.orchestra.run.smithi031.stdout: Package Architecture Version Repository Size 2023-10-01T14:42:33.608 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2023-10-01T14:42:33.608 INFO:teuthology.orchestra.run.smithi031.stdout:Installing: 2023-10-01T14:42:33.609 INFO:teuthology.orchestra.run.smithi031.stdout: kernel x86_64 4.18.0-514.el8 baseos 10 M 2023-10-01T14:42:33.609 INFO:teuthology.orchestra.run.smithi031.stdout:Installing dependencies: 2023-10-01T14:42:33.609 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-core x86_64 4.18.0-514.el8 baseos 43 M 2023-10-01T14:42:33.609 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-modules x86_64 4.18.0-514.el8 baseos 35 M 2023-10-01T14:42:33.609 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:42:33.609 INFO:teuthology.orchestra.run.smithi031.stdout:Transaction Summary 2023-10-01T14:42:33.609 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2023-10-01T14:42:33.609 INFO:teuthology.orchestra.run.smithi031.stdout:Install 3 Packages 2023-10-01T14:42:33.609 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:42:33.610 INFO:teuthology.orchestra.run.smithi031.stdout:Total download size: 88 M 2023-10-01T14:42:33.610 INFO:teuthology.orchestra.run.smithi031.stdout:Installed size: 96 M 2023-10-01T14:42:33.610 INFO:teuthology.orchestra.run.smithi031.stdout:Downloading Packages: 2023-10-01T14:42:33.658 INFO:teuthology.orchestra.run.smithi099.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-10-01T14:42:33.658 INFO:teuthology.orchestra.run.smithi099.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-10-01T14:42:33.907 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-10-01T14:42:33.907 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-10-01T14:42:33.908 INFO:teuthology.orchestra.run.smithi099.stdout: Package Architecture Version Repository Size 2023-10-01T14:42:33.908 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-10-01T14:42:33.908 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-10-01T14:42:33.908 INFO:teuthology.orchestra.run.smithi099.stdout: kernel x86_64 4.18.0-514.el8 baseos 10 M 2023-10-01T14:42:33.908 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2023-10-01T14:42:33.908 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-core x86_64 4.18.0-514.el8 baseos 43 M 2023-10-01T14:42:33.908 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-modules x86_64 4.18.0-514.el8 baseos 35 M 2023-10-01T14:42:33.909 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-10-01T14:42:33.909 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-10-01T14:42:33.909 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-10-01T14:42:33.910 INFO:teuthology.orchestra.run.smithi099.stdout:Install 3 Packages 2023-10-01T14:42:33.910 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-10-01T14:42:33.910 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 88 M 2023-10-01T14:42:33.910 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 96 M 2023-10-01T14:42:33.910 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-10-01T14:42:34.001 INFO:teuthology.orchestra.run.smithi031.stdout:(1/3): kernel-4.18.0-514.el8.x86_64.rpm 29 MB/s | 10 MB 00:00 2023-10-01T14:42:34.114 INFO:teuthology.orchestra.run.smithi135.stdout:(1/3): kernel-4.18.0-514.el8.x86_64.rpm 25 MB/s | 10 MB 00:00 2023-10-01T14:42:34.511 INFO:teuthology.orchestra.run.smithi099.stdout:(1/3): kernel-4.18.0-514.el8.x86_64.rpm 24 MB/s | 10 MB 00:00 2023-10-01T14:42:34.702 INFO:teuthology.orchestra.run.smithi135.stdout:(2/3): kernel-modules-4.18.0-514.el8.x86_64.rpm 36 MB/s | 35 MB 00:00 2023-10-01T14:42:34.851 INFO:teuthology.orchestra.run.smithi031.stdout:(2/3): kernel-modules-4.18.0-514.el8.x86_64.rpm 30 MB/s | 35 MB 00:01 2023-10-01T14:42:35.309 INFO:teuthology.orchestra.run.smithi031.stdout:(3/3): kernel-core-4.18.0-514.el8.x86_64.rpm 26 MB/s | 43 MB 00:01 2023-10-01T14:42:35.310 INFO:teuthology.orchestra.run.smithi031.stdout:-------------------------------------------------------------------------------- 2023-10-01T14:42:35.310 INFO:teuthology.orchestra.run.smithi031.stdout:Total 52 MB/s | 88 MB 00:01 2023-10-01T14:42:35.361 INFO:teuthology.orchestra.run.smithi099.stdout:(2/3): kernel-modules-4.18.0-514.el8.x86_64.rpm 28 MB/s | 35 MB 00:01 2023-10-01T14:42:35.753 INFO:teuthology.orchestra.run.smithi099.stdout:(3/3): kernel-core-4.18.0-514.el8.x86_64.rpm 26 MB/s | 43 MB 00:01 2023-10-01T14:42:35.753 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-10-01T14:42:35.754 INFO:teuthology.orchestra.run.smithi099.stdout:Total 48 MB/s | 88 MB 00:01 2023-10-01T14:42:35.840 INFO:teuthology.orchestra.run.smithi135.stdout:(3/3): kernel-core-4.18.0-514.el8.x86_64.rpm 20 MB/s | 43 MB 00:02 2023-10-01T14:42:35.840 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-10-01T14:42:35.840 INFO:teuthology.orchestra.run.smithi135.stdout:Total 38 MB/s | 88 MB 00:02 2023-10-01T14:42:36.012 INFO:teuthology.orchestra.run.smithi031.stdout:Running transaction check 2023-10-01T14:42:36.423 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-10-01T14:42:36.443 INFO:teuthology.orchestra.run.smithi031.stdout:Transaction check succeeded. 2023-10-01T14:42:36.443 INFO:teuthology.orchestra.run.smithi031.stdout:Running transaction test 2023-10-01T14:42:36.510 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-10-01T14:42:36.861 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-10-01T14:42:36.861 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-10-01T14:42:36.930 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-10-01T14:42:36.930 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-10-01T14:42:37.725 INFO:teuthology.orchestra.run.smithi031.stdout:Transaction test succeeded. 2023-10-01T14:42:37.745 INFO:teuthology.orchestra.run.smithi031.stdout:Running transaction 2023-10-01T14:42:38.067 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-10-01T14:42:38.082 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-10-01T14:42:38.158 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-10-01T14:42:38.188 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-10-01T14:42:40.631 INFO:teuthology.orchestra.run.smithi031.stdout: Preparing : 1/1 2023-10-01T14:42:40.826 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-10-01T14:42:40.932 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-10-01T14:42:41.149 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : kernel-core-4.18.0-514.el8.x86_64 1/3 2023-10-01T14:42:41.384 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : kernel-core-4.18.0-514.el8.x86_64 1/3 2023-10-01T14:42:41.472 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : kernel-core-4.18.0-514.el8.x86_64 1/3 2023-10-01T14:42:44.871 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-core-4.18.0-514.el8.x86_64 1/3 2023-10-01T14:42:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-core-4.18.0-514.el8.x86_64 1/3 2023-10-01T14:42:44.996 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-core-4.18.0-514.el8.x86_64 1/3 2023-10-01T14:42:45.232 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : kernel-modules-4.18.0-514.el8.x86_64 2/3 2023-10-01T14:42:45.317 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : kernel-modules-4.18.0-514.el8.x86_64 2/3 2023-10-01T14:42:45.463 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : kernel-modules-4.18.0-514.el8.x86_64 2/3 2023-10-01T14:42:48.692 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-modules-4.18.0-514.el8.x86_64 2/3 2023-10-01T14:42:48.837 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-modules-4.18.0-514.el8.x86_64 2/3 2023-10-01T14:42:48.853 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-modules-4.18.0-514.el8.x86_64 2/3 2023-10-01T14:42:48.981 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : kernel-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:42:49.137 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : kernel-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:42:49.141 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : kernel-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:43:10.728 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-core-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:43:10.753 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-modules-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:43:11.069 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-core-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:43:11.093 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-modules-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:43:11.100 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:43:11.100 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : kernel-4.18.0-514.el8.x86_64 1/3 2023-10-01T14:43:11.101 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : kernel-core-4.18.0-514.el8.x86_64 2/3 2023-10-01T14:43:11.450 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:43:11.450 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : kernel-4.18.0-514.el8.x86_64 1/3 2023-10-01T14:43:11.450 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : kernel-core-4.18.0-514.el8.x86_64 2/3 2023-10-01T14:43:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : kernel-modules-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:43:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-10-01T14:43:11.856 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-10-01T14:43:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-4.18.0-514.el8.x86_64 kernel-core-4.18.0-514.el8.x86_64 2023-10-01T14:43:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-modules-4.18.0-514.el8.x86_64 2023-10-01T14:43:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-10-01T14:43:11.856 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-10-01T14:43:12.002 DEBUG:teuthology.orchestra.run.smithi099:> rpm -q kernel | sort -rV | head -n 1 2023-10-01T14:43:12.214 INFO:teuthology.orchestra.run.smithi099.stdout:kernel-4.18.0-514.el8.x86_64 2023-10-01T14:43:12.214 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-514.el8.x86_64 2023-10-01T14:43:12.214 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-514.el8.x86_64 2023-10-01T14:43:12.215 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-01T14:43:12.215 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-01T14:43:12.215 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2023-10-01T14:43:12.215 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi099.front.sepia.ceph.com, path=None, version=distro) 2023-10-01T14:43:12.215 DEBUG:teuthology.orchestra.run.smithi099:> rpm -q kernel | sort -rV | head -n 1 2023-10-01T14:43:12.379 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-core-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:43:12.403 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-modules-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:43:12.422 INFO:teuthology.orchestra.run.smithi099.stdout:kernel-4.18.0-514.el8.x86_64 2023-10-01T14:43:12.423 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-514.el8.x86_64 2023-10-01T14:43:12.423 DEBUG:teuthology.orchestra.run.smithi099:> sudo rpm -qi grub2-tools 2023-10-01T14:43:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:Name : grub2-tools 2023-10-01T14:43:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:Epoch : 1 2023-10-01T14:43:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:Version : 2.02 2023-10-01T14:43:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:Release : 129.el8 2023-10-01T14:43:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:Architecture: x86_64 2023-10-01T14:43:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-10-01T14:43:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:Group : System Environment/Base 2023-10-01T14:43:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:Size : 9265586 2023-10-01T14:43:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:License : GPLv3+ 2023-10-01T14:43:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-10-01T14:43:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-10-01T14:43:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-10-01T14:43:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:Build Host : kbuilder.bsys.centos.org 2023-10-01T14:43:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:Relocations : (not relocatable) 2023-10-01T14:43:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:Packager : CentOS BuildSystem 2023-10-01T14:43:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:Vendor : CentOS 2023-10-01T14:43:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:URL : http://www.gnu.org/software/grub/ 2023-10-01T14:43:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:Summary : Support tools for GRUB. 2023-10-01T14:43:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:Description : 2023-10-01T14:43:12.458 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-10-01T14:43:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-10-01T14:43:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:customizable bootloader with modular architecture. It supports a rich 2023-10-01T14:43:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:variety of kernel formats, file systems, computer architectures and 2023-10-01T14:43:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:hardware devices. 2023-10-01T14:43:12.458 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-10-01T14:43:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:This subpackage provides tools for support of all platforms. 2023-10-01T14:43:12.459 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-10-01T14:43:12.459 INFO:teuthology.task.kernel:Updating grub on smithi099 to boot 4.18.0-514.el8.x86_64 2023-10-01T14:43:12.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-10-01T14:43:12.753 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:43:12.753 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : kernel-4.18.0-514.el8.x86_64 1/3 2023-10-01T14:43:12.753 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : kernel-core-4.18.0-514.el8.x86_64 2/3 2023-10-01T14:43:12.756 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : kernel-modules-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:43:12.756 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:43:12.756 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-10-01T14:43:12.757 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-4.18.0-514.el8.x86_64 kernel-core-4.18.0-514.el8.x86_64 2023-10-01T14:43:12.757 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-modules-4.18.0-514.el8.x86_64 2023-10-01T14:43:12.757 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:43:12.757 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-10-01T14:43:12.883 DEBUG:teuthology.orchestra.run.smithi135:> rpm -q kernel | sort -rV | head -n 1 2023-10-01T14:43:13.096 INFO:teuthology.orchestra.run.smithi135.stdout:kernel-4.18.0-514.el8.x86_64 2023-10-01T14:43:13.096 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-514.el8.x86_64 2023-10-01T14:43:13.096 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-514.el8.x86_64 2023-10-01T14:43:13.096 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-01T14:43:13.096 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-01T14:43:13.097 INFO:teuthology.task.kernel:Installing distro kernel on mon.c... 2023-10-01T14:43:13.097 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi135.front.sepia.ceph.com, path=None, version=distro) 2023-10-01T14:43:13.097 DEBUG:teuthology.orchestra.run.smithi135:> rpm -q kernel | sort -rV | head -n 1 2023-10-01T14:43:13.305 INFO:teuthology.orchestra.run.smithi135.stdout:kernel-4.18.0-514.el8.x86_64 2023-10-01T14:43:13.305 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-514.el8.x86_64 2023-10-01T14:43:13.306 DEBUG:teuthology.orchestra.run.smithi135:> sudo rpm -qi grub2-tools 2023-10-01T14:43:13.339 INFO:teuthology.orchestra.run.smithi135.stdout:Name : grub2-tools 2023-10-01T14:43:13.339 INFO:teuthology.orchestra.run.smithi135.stdout:Epoch : 1 2023-10-01T14:43:13.339 INFO:teuthology.orchestra.run.smithi135.stdout:Version : 2.02 2023-10-01T14:43:13.339 INFO:teuthology.orchestra.run.smithi135.stdout:Release : 129.el8 2023-10-01T14:43:13.339 INFO:teuthology.orchestra.run.smithi135.stdout:Architecture: x86_64 2023-10-01T14:43:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-10-01T14:43:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:Group : System Environment/Base 2023-10-01T14:43:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:Size : 9265586 2023-10-01T14:43:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:License : GPLv3+ 2023-10-01T14:43:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-10-01T14:43:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-10-01T14:43:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-10-01T14:43:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:Build Host : kbuilder.bsys.centos.org 2023-10-01T14:43:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:Relocations : (not relocatable) 2023-10-01T14:43:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:Packager : CentOS BuildSystem 2023-10-01T14:43:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:Vendor : CentOS 2023-10-01T14:43:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:URL : http://www.gnu.org/software/grub/ 2023-10-01T14:43:13.341 INFO:teuthology.orchestra.run.smithi135.stdout:Summary : Support tools for GRUB. 2023-10-01T14:43:13.341 INFO:teuthology.orchestra.run.smithi135.stdout:Description : 2023-10-01T14:43:13.341 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:43:13.341 INFO:teuthology.orchestra.run.smithi135.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-10-01T14:43:13.341 INFO:teuthology.orchestra.run.smithi135.stdout:customizable bootloader with modular architecture. It supports a rich 2023-10-01T14:43:13.341 INFO:teuthology.orchestra.run.smithi135.stdout:variety of kernel formats, file systems, computer architectures and 2023-10-01T14:43:13.341 INFO:teuthology.orchestra.run.smithi135.stdout:hardware devices. 2023-10-01T14:43:13.341 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:43:13.341 INFO:teuthology.orchestra.run.smithi135.stdout:This subpackage provides tools for support of all platforms. 2023-10-01T14:43:13.342 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-10-01T14:43:13.342 INFO:teuthology.task.kernel:Updating grub on smithi135 to boot 4.18.0-514.el8.x86_64 2023-10-01T14:43:13.342 DEBUG:teuthology.orchestra.run.smithi135:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-10-01T14:43:13.506 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : kernel-modules-4.18.0-514.el8.x86_64 3/3 2023-10-01T14:43:13.506 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:43:13.506 INFO:teuthology.orchestra.run.smithi031.stdout:Installed: 2023-10-01T14:43:13.506 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-4.18.0-514.el8.x86_64 kernel-core-4.18.0-514.el8.x86_64 2023-10-01T14:43:13.506 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-modules-4.18.0-514.el8.x86_64 2023-10-01T14:43:13.507 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:43:13.507 INFO:teuthology.orchestra.run.smithi031.stdout:Complete! 2023-10-01T14:43:13.652 DEBUG:teuthology.orchestra.run.smithi031:> rpm -q kernel | sort -rV | head -n 1 2023-10-01T14:43:13.843 INFO:teuthology.orchestra.run.smithi099.stderr:Generating grub configuration file ... 2023-10-01T14:43:13.878 INFO:teuthology.orchestra.run.smithi031.stdout:kernel-4.18.0-514.el8.x86_64 2023-10-01T14:43:13.878 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-514.el8.x86_64 2023-10-01T14:43:13.878 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-514.el8.x86_64 2023-10-01T14:43:13.878 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-01T14:43:13.878 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-01T14:43:13.879 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2023-10-01T14:43:13.879 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi031.front.sepia.ceph.com, path=None, version=distro) 2023-10-01T14:43:13.879 DEBUG:teuthology.orchestra.run.smithi031:> rpm -q kernel | sort -rV | head -n 1 2023-10-01T14:43:14.096 INFO:teuthology.orchestra.run.smithi031.stdout:kernel-4.18.0-514.el8.x86_64 2023-10-01T14:43:14.097 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-514.el8.x86_64 2023-10-01T14:43:14.097 DEBUG:teuthology.orchestra.run.smithi031:> sudo rpm -qi grub2-tools 2023-10-01T14:43:14.131 INFO:teuthology.orchestra.run.smithi031.stdout:Name : grub2-tools 2023-10-01T14:43:14.131 INFO:teuthology.orchestra.run.smithi031.stdout:Epoch : 1 2023-10-01T14:43:14.132 INFO:teuthology.orchestra.run.smithi031.stdout:Version : 2.02 2023-10-01T14:43:14.132 INFO:teuthology.orchestra.run.smithi031.stdout:Release : 129.el8 2023-10-01T14:43:14.132 INFO:teuthology.orchestra.run.smithi031.stdout:Architecture: x86_64 2023-10-01T14:43:14.132 INFO:teuthology.orchestra.run.smithi031.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-10-01T14:43:14.132 INFO:teuthology.orchestra.run.smithi031.stdout:Group : System Environment/Base 2023-10-01T14:43:14.132 INFO:teuthology.orchestra.run.smithi031.stdout:Size : 9265586 2023-10-01T14:43:14.132 INFO:teuthology.orchestra.run.smithi031.stdout:License : GPLv3+ 2023-10-01T14:43:14.132 INFO:teuthology.orchestra.run.smithi031.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-10-01T14:43:14.132 INFO:teuthology.orchestra.run.smithi031.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-10-01T14:43:14.132 INFO:teuthology.orchestra.run.smithi031.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-10-01T14:43:14.132 INFO:teuthology.orchestra.run.smithi031.stdout:Build Host : kbuilder.bsys.centos.org 2023-10-01T14:43:14.132 INFO:teuthology.orchestra.run.smithi031.stdout:Relocations : (not relocatable) 2023-10-01T14:43:14.133 INFO:teuthology.orchestra.run.smithi031.stdout:Packager : CentOS BuildSystem 2023-10-01T14:43:14.133 INFO:teuthology.orchestra.run.smithi031.stdout:Vendor : CentOS 2023-10-01T14:43:14.133 INFO:teuthology.orchestra.run.smithi031.stdout:URL : http://www.gnu.org/software/grub/ 2023-10-01T14:43:14.133 INFO:teuthology.orchestra.run.smithi031.stdout:Summary : Support tools for GRUB. 2023-10-01T14:43:14.133 INFO:teuthology.orchestra.run.smithi031.stdout:Description : 2023-10-01T14:43:14.133 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:43:14.133 INFO:teuthology.orchestra.run.smithi031.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-10-01T14:43:14.133 INFO:teuthology.orchestra.run.smithi031.stdout:customizable bootloader with modular architecture. It supports a rich 2023-10-01T14:43:14.133 INFO:teuthology.orchestra.run.smithi031.stdout:variety of kernel formats, file systems, computer architectures and 2023-10-01T14:43:14.133 INFO:teuthology.orchestra.run.smithi031.stdout:hardware devices. 2023-10-01T14:43:14.133 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:43:14.133 INFO:teuthology.orchestra.run.smithi031.stdout:This subpackage provides tools for support of all platforms. 2023-10-01T14:43:14.134 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-10-01T14:43:14.134 INFO:teuthology.task.kernel:Updating grub on smithi031 to boot 4.18.0-514.el8.x86_64 2023-10-01T14:43:14.134 DEBUG:teuthology.orchestra.run.smithi031:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-10-01T14:43:14.908 INFO:teuthology.orchestra.run.smithi135.stderr:Generating grub configuration file ... 2023-10-01T14:43:15.427 INFO:teuthology.orchestra.run.smithi031.stderr:Generating grub configuration file ... 2023-10-01T14:43:16.260 INFO:teuthology.orchestra.run.smithi099.stderr:done 2023-10-01T14:43:16.262 DEBUG:teuthology.orchestra.run.smithi099:> mktemp 2023-10-01T14:43:16.287 INFO:teuthology.orchestra.run.smithi099.stdout:/tmp/tmp.AaksFwvF51 2023-10-01T14:43:16.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.AaksFwvF51 2023-10-01T14:43:16.362 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 0666 /tmp/tmp.AaksFwvF51 2023-10-01T14:43:16.502 DEBUG:teuthology.orchestra.remote:smithi099:/tmp/tmp.AaksFwvF51 is 6KB 2023-10-01T14:43:16.550 DEBUG:teuthology.orchestra.run.smithi099:> rm -fr /tmp/tmp.AaksFwvF51 2023-10-01T14:43:16.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo /bin/ls /boot/loader/entries || true 2023-10-01T14:43:16.650 INFO:teuthology.orchestra.run.smithi099.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-10-01T14:43:16.650 INFO:teuthology.orchestra.run.smithi099.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-10-01T14:43:16.650 INFO:teuthology.orchestra.run.smithi099.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-10-01T14:43:16.650 INFO:teuthology.orchestra.run.smithi099.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-514.el8.x86_64.conf 2023-10-01T14:43:16.651 DEBUG:teuthology.orchestra.run.smithi099:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-514.el8.x86_64 2023-10-01T14:43:16.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo shutdown -r now 2023-10-01T14:43:17.151 INFO:teuthology.orchestra.run.smithi135.stderr:done 2023-10-01T14:43:17.154 DEBUG:teuthology.orchestra.run.smithi135:> mktemp 2023-10-01T14:43:17.180 INFO:teuthology.orchestra.run.smithi135.stdout:/tmp/tmp.oUD7wvQ7zL 2023-10-01T14:43:17.180 DEBUG:teuthology.orchestra.run.smithi135:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.oUD7wvQ7zL 2023-10-01T14:43:17.254 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod 0666 /tmp/tmp.oUD7wvQ7zL 2023-10-01T14:43:17.403 DEBUG:teuthology.orchestra.remote:smithi135:/tmp/tmp.oUD7wvQ7zL is 6KB 2023-10-01T14:43:17.450 DEBUG:teuthology.orchestra.run.smithi135:> rm -fr /tmp/tmp.oUD7wvQ7zL 2023-10-01T14:43:17.468 DEBUG:teuthology.orchestra.run.smithi135:> sudo /bin/ls /boot/loader/entries || true 2023-10-01T14:43:17.533 INFO:teuthology.orchestra.run.smithi135.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-10-01T14:43:17.533 INFO:teuthology.orchestra.run.smithi135.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-10-01T14:43:17.533 INFO:teuthology.orchestra.run.smithi135.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-10-01T14:43:17.533 INFO:teuthology.orchestra.run.smithi135.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-514.el8.x86_64.conf 2023-10-01T14:43:17.534 DEBUG:teuthology.orchestra.run.smithi135:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-514.el8.x86_64 2023-10-01T14:43:17.550 INFO:teuthology.orchestra.run.smithi031.stderr:done 2023-10-01T14:43:17.552 DEBUG:teuthology.orchestra.run.smithi031:> mktemp 2023-10-01T14:43:17.580 INFO:teuthology.orchestra.run.smithi031.stdout:/tmp/tmp.XqeARIkct5 2023-10-01T14:43:17.581 DEBUG:teuthology.orchestra.run.smithi031:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.XqeARIkct5 2023-10-01T14:43:17.653 DEBUG:teuthology.orchestra.run.smithi031:> sudo chmod 0666 /tmp/tmp.XqeARIkct5 2023-10-01T14:43:17.690 DEBUG:teuthology.orchestra.run.smithi135:> sudo shutdown -r now 2023-10-01T14:43:17.795 DEBUG:teuthology.orchestra.remote:smithi031:/tmp/tmp.XqeARIkct5 is 6KB 2023-10-01T14:43:17.842 DEBUG:teuthology.orchestra.run.smithi031:> rm -fr /tmp/tmp.XqeARIkct5 2023-10-01T14:43:17.860 DEBUG:teuthology.orchestra.run.smithi031:> sudo /bin/ls /boot/loader/entries || true 2023-10-01T14:43:17.926 INFO:teuthology.orchestra.run.smithi031.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-10-01T14:43:17.926 INFO:teuthology.orchestra.run.smithi031.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-10-01T14:43:17.926 INFO:teuthology.orchestra.run.smithi031.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-10-01T14:43:17.927 INFO:teuthology.orchestra.run.smithi031.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-514.el8.x86_64.conf 2023-10-01T14:43:17.928 DEBUG:teuthology.orchestra.run.smithi031:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-514.el8.x86_64 2023-10-01T14:43:18.097 DEBUG:teuthology.orchestra.run.smithi031:> sudo shutdown -r now 2023-10-01T14:43:46.779 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2023-10-01T14:43:46.779 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2023-10-01T14:43:46.780 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:43:47.693 INFO:teuthology.task.kernel:Checking client mon.c for new kernel version... 2023-10-01T14:43:47.693 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2023-10-01T14:43:47.693 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:43:48.101 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2023-10-01T14:43:48.101 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi031.front.sepia.ceph.com' 2023-10-01T14:43:48.101 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi031.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:44:05.118 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.99 2023-10-01T14:44:06.138 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.135 2023-10-01T14:44:06.654 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.31 2023-10-01T14:44:14.127 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2023-10-01T14:44:14.128 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:44:15.141 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2023-10-01T14:44:15.141 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:44:15.655 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi031.front.sepia.ceph.com' 2023-10-01T14:44:15.656 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi031.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:44:17.182 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.99 2023-10-01T14:44:18.206 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.135 2023-10-01T14:44:29.183 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2023-10-01T14:44:29.184 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:44:30.209 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2023-10-01T14:44:30.209 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:44:34.042 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.31 2023-10-01T14:44:46.055 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi031.front.sepia.ceph.com' 2023-10-01T14:44:46.055 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi031.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:44:46.525 DEBUG:teuthology.orchestra.run.smithi031:> true 2023-10-01T14:44:46.806 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi031.front.sepia.ceph.com' 2023-10-01T14:44:46.806 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "4.18.0-514.el8.x86_64"... 2023-10-01T14:44:46.807 DEBUG:teuthology.orchestra.run.smithi031:> uname -r 2023-10-01T14:44:46.829 INFO:teuthology.orchestra.run.smithi031.stdout:4.18.0-514.el8.x86_64 2023-10-01T14:44:46.829 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-514.el8.x86_64 vs 4.18.0-514.el8.x86_64 2023-10-01T14:44:46.829 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-01T14:44:46.829 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-10-01T14:44:47.610 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.99 2023-10-01T14:44:47.830 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2023-10-01T14:44:47.831 DEBUG:teuthology.orchestra.run.smithi031:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-01T14:44:47.898 INFO:teuthology.orchestra.run.smithi031.stdout:ttyS1 2023-10-01T14:44:47.923 DEBUG:teuthology.parallel:result is None 2023-10-01T14:44:48.634 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.135 2023-10-01T14:45:02.612 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2023-10-01T14:45:02.612 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:45:03.091 DEBUG:teuthology.orchestra.run.smithi099:> true 2023-10-01T14:45:03.338 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi099.front.sepia.ceph.com' 2023-10-01T14:45:03.338 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "4.18.0-514.el8.x86_64"... 2023-10-01T14:45:03.338 DEBUG:teuthology.orchestra.run.smithi099:> uname -r 2023-10-01T14:45:03.355 INFO:teuthology.orchestra.run.smithi099.stdout:4.18.0-514.el8.x86_64 2023-10-01T14:45:03.355 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-514.el8.x86_64 vs 4.18.0-514.el8.x86_64 2023-10-01T14:45:03.356 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-01T14:45:03.356 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-10-01T14:45:03.635 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2023-10-01T14:45:03.635 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:45:04.004 DEBUG:teuthology.orchestra.run.smithi135:> true 2023-10-01T14:45:04.335 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi135.front.sepia.ceph.com' 2023-10-01T14:45:04.335 INFO:teuthology.task.kernel:Checking kernel version of mon.c, want "4.18.0-514.el8.x86_64"... 2023-10-01T14:45:04.336 DEBUG:teuthology.orchestra.run.smithi135:> uname -r 2023-10-01T14:45:04.353 INFO:teuthology.orchestra.run.smithi135.stdout:4.18.0-514.el8.x86_64 2023-10-01T14:45:04.353 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-514.el8.x86_64 vs 4.18.0-514.el8.x86_64 2023-10-01T14:45:04.353 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-01T14:45:04.353 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-10-01T14:45:04.356 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2023-10-01T14:45:04.357 DEBUG:teuthology.orchestra.run.smithi099:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-01T14:45:04.615 INFO:teuthology.orchestra.run.smithi099.stdout:ttyS1 2023-10-01T14:45:04.627 DEBUG:teuthology.parallel:result is None 2023-10-01T14:45:05.354 INFO:teuthology.task.kernel:Enabling kdb on mon.c... 2023-10-01T14:45:05.354 DEBUG:teuthology.orchestra.run.smithi135:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-01T14:45:06.131 INFO:teuthology.orchestra.run.smithi135.stdout:ttyS1 2023-10-01T14:45:06.161 DEBUG:teuthology.parallel:result is None 2023-10-01T14:45:06.162 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-01T14:45:06.168 INFO:teuthology.task.internal:Creating test directory... 2023-10-01T14:45:06.169 DEBUG:teuthology.orchestra.run.smithi031:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-01T14:45:06.172 DEBUG:teuthology.orchestra.run.smithi099:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-01T14:45:06.242 DEBUG:teuthology.orchestra.run.smithi135:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-01T14:45:06.624 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-01T14:45:06.631 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-01T14:45:06.641 INFO:teuthology.task.internal:Creating archive directory... 2023-10-01T14:45:06.641 DEBUG:teuthology.orchestra.run.smithi031:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-01T14:45:06.643 DEBUG:teuthology.orchestra.run.smithi099:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-01T14:45:06.646 DEBUG:teuthology.orchestra.run.smithi135:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-01T14:45:06.695 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-01T14:45:06.702 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-01T14:45:06.702 DEBUG:teuthology.orchestra.run.smithi031:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-01T14:45:06.715 DEBUG:teuthology.orchestra.run.smithi099:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-01T14:45:06.717 DEBUG:teuthology.orchestra.run.smithi135:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-01T14:45:06.754 INFO:teuthology.orchestra.run.smithi031.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T14:45:06.759 INFO:teuthology.orchestra.run.smithi135.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T14:45:06.767 INFO:teuthology.orchestra.run.smithi031.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T14:45:06.770 INFO:teuthology.orchestra.run.smithi135.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T14:45:06.839 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T14:45:06.849 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T14:45:06.850 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-01T14:45:06.857 INFO:teuthology.task.internal:Configuring sudo... 2023-10-01T14:45:06.857 DEBUG:teuthology.orchestra.run.smithi031:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-01T14:45:06.860 DEBUG:teuthology.orchestra.run.smithi099:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-01T14:45:06.862 DEBUG:teuthology.orchestra.run.smithi135:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-01T14:45:06.893 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-01T14:45:06.902 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-01T14:45:06.902 DEBUG:teuthology.orchestra.run.smithi031:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-01T14:45:06.936 DEBUG:teuthology.orchestra.run.smithi099:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-01T14:45:06.938 DEBUG:teuthology.orchestra.run.smithi135:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-01T14:45:06.956 DEBUG:teuthology.orchestra.run.smithi031:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-01T14:45:07.047 DEBUG:teuthology.orchestra.run.smithi031:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-01T14:45:07.114 DEBUG:teuthology.orchestra.run.smithi031:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-01T14:45:07.162 DEBUG:teuthology.orchestra.run.smithi031:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-01T14:45:07.214 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2023-10-01T14:45:07.214 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-01T14:45:07.289 DEBUG:teuthology.orchestra.run.smithi099:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-01T14:45:07.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-01T14:45:07.563 DEBUG:teuthology.orchestra.run.smithi099:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-01T14:45:07.647 DEBUG:teuthology.orchestra.run.smithi099:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-01T14:45:07.690 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-10-01T14:45:07.691 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-01T14:45:07.757 DEBUG:teuthology.orchestra.run.smithi135:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-01T14:45:07.803 DEBUG:teuthology.orchestra.run.smithi135:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-01T14:45:07.852 DEBUG:teuthology.orchestra.run.smithi135:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-01T14:45:07.938 DEBUG:teuthology.orchestra.run.smithi135:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-01T14:45:07.978 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-10-01T14:45:07.979 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-01T14:45:08.046 DEBUG:teuthology.orchestra.run.smithi031:> sudo service rsyslog restart 2023-10-01T14:45:08.048 DEBUG:teuthology.orchestra.run.smithi099:> sudo service rsyslog restart 2023-10-01T14:45:08.050 DEBUG:teuthology.orchestra.run.smithi135:> sudo service rsyslog restart 2023-10-01T14:45:08.112 INFO:teuthology.orchestra.run.smithi099.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-01T14:45:08.135 INFO:teuthology.orchestra.run.smithi031.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-01T14:45:08.155 INFO:teuthology.orchestra.run.smithi135.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-01T14:45:08.620 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-01T14:45:08.628 INFO:teuthology.task.internal:Starting timer... 2023-10-01T14:45:08.628 INFO:teuthology.run_tasks:Running task pcp... 2023-10-01T14:45:08.637 INFO:teuthology.run_tasks:Running task selinux... 2023-10-01T14:45:08.645 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-10-01T14:45:08.646 DEBUG:teuthology.orchestra.run.smithi031:> sudo service auditd rotate 2023-10-01T14:45:08.736 INFO:teuthology.orchestra.run.smithi031.stdout:Rotating logs: 2023-10-01T14:45:08.738 DEBUG:teuthology.orchestra.run.smithi099:> sudo service auditd rotate 2023-10-01T14:45:08.808 INFO:teuthology.orchestra.run.smithi099.stdout:Rotating logs: 2023-10-01T14:45:08.810 DEBUG:teuthology.orchestra.run.smithi135:> sudo service auditd rotate 2023-10-01T14:45:08.878 INFO:teuthology.orchestra.run.smithi135.stdout:Rotating logs: 2023-10-01T14:45:08.879 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-01T14:45:08.880 DEBUG:teuthology.orchestra.run.smithi031:> /usr/sbin/getenforce 2023-10-01T14:45:08.928 INFO:teuthology.orchestra.run.smithi031.stdout:Permissive 2023-10-01T14:45:08.928 DEBUG:teuthology.orchestra.run.smithi099:> /usr/sbin/getenforce 2023-10-01T14:45:08.956 INFO:teuthology.orchestra.run.smithi099.stdout:Permissive 2023-10-01T14:45:08.956 DEBUG:teuthology.orchestra.run.smithi135:> /usr/sbin/getenforce 2023-10-01T14:45:08.981 INFO:teuthology.orchestra.run.smithi135.stdout:Permissive 2023-10-01T14:45:08.981 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi031.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi099.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi135.front.sepia.ceph.com': 'permissive'} 2023-10-01T14:45:08.981 DEBUG:teuthology.orchestra.run.smithi031:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-01T14:45:09.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T14:45:09.016 DEBUG:teuthology.orchestra.run.smithi099:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-01T14:45:09.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T14:45:09.043 DEBUG:teuthology.orchestra.run.smithi135:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-01T14:45:09.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T14:45:09.071 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-01T14:45:09.071 DEBUG:teuthology.orchestra.run.smithi031:> sudo /usr/sbin/setenforce permissive 2023-10-01T14:45:09.106 DEBUG:teuthology.orchestra.run.smithi099:> sudo /usr/sbin/setenforce permissive 2023-10-01T14:45:09.133 DEBUG:teuthology.orchestra.run.smithi135:> sudo /usr/sbin/setenforce permissive 2023-10-01T14:45:09.160 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-01T14:45:09.179 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-10-01T14:45:09.185 INFO:teuthology.repo_utils:Fetching main from origin 2023-10-01T14:45:09.290 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-01T14:45:09.305 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-01T14:45:09.306 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi031.front.sepia.ceph.com,smithi099.front.sepia.ceph.com,smithi135.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-01T14:53:07.299 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi031.front.sepia.ceph.com'), Remote(name='ubuntu@smithi099.front.sepia.ceph.com'), Remote(name='ubuntu@smithi135.front.sepia.ceph.com')] 2023-10-01T14:53:07.300 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi031.front.sepia.ceph.com' 2023-10-01T14:53:07.301 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi031.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:53:07.374 DEBUG:teuthology.orchestra.run.smithi031:> true 2023-10-01T14:53:07.450 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi031.front.sepia.ceph.com' 2023-10-01T14:53:07.450 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2023-10-01T14:53:07.451 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:53:07.517 DEBUG:teuthology.orchestra.run.smithi099:> true 2023-10-01T14:53:07.547 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi099.front.sepia.ceph.com' 2023-10-01T14:53:07.547 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2023-10-01T14:53:07.547 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T14:53:07.616 DEBUG:teuthology.orchestra.run.smithi135:> true 2023-10-01T14:53:07.643 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi135.front.sepia.ceph.com' 2023-10-01T14:53:07.644 INFO:teuthology.run_tasks:Running task clock... 2023-10-01T14:53:07.654 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-01T14:53:07.654 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-01T14:53:07.654 DEBUG:teuthology.orchestra.run.smithi031:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T14:53:07.657 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-01T14:53:07.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T14:53:07.659 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-01T14:53:07.659 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T14:53:07.692 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-01T14:53:07.695 INFO:teuthology.orchestra.run.smithi031.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-01T14:53:07.706 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-01T14:53:07.712 INFO:teuthology.orchestra.run.smithi031.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-01T14:53:07.718 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-01T14:53:07.733 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-01T14:53:07.767 INFO:teuthology.orchestra.run.smithi099.stderr:sudo: ntpd: command not found 2023-10-01T14:53:07.779 INFO:teuthology.orchestra.run.smithi099.stdout:506 Cannot talk to daemon 2023-10-01T14:53:07.791 INFO:teuthology.orchestra.run.smithi031.stderr:sudo: ntpd: command not found 2023-10-01T14:53:07.792 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-01T14:53:07.801 INFO:teuthology.orchestra.run.smithi135.stderr:sudo: ntpd: command not found 2023-10-01T14:53:07.805 INFO:teuthology.orchestra.run.smithi031.stdout:506 Cannot talk to daemon 2023-10-01T14:53:07.805 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-01T14:53:07.813 INFO:teuthology.orchestra.run.smithi135.stdout:506 Cannot talk to daemon 2023-10-01T14:53:07.822 INFO:teuthology.orchestra.run.smithi031.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-01T14:53:07.827 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-01T14:53:07.837 INFO:teuthology.orchestra.run.smithi031.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-01T14:53:07.841 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-01T14:53:07.885 INFO:teuthology.orchestra.run.smithi099.stderr:bash: ntpq: command not found 2023-10-01T14:53:07.889 INFO:teuthology.orchestra.run.smithi099.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-01T14:53:07.889 INFO:teuthology.orchestra.run.smithi099.stdout:=============================================================================== 2023-10-01T14:53:07.889 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T14:53:07.889 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T14:53:07.889 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T14:53:07.890 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T14:53:07.909 INFO:teuthology.orchestra.run.smithi135.stderr:bash: ntpq: command not found 2023-10-01T14:53:07.913 INFO:teuthology.orchestra.run.smithi135.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-01T14:53:07.913 INFO:teuthology.orchestra.run.smithi135.stdout:=============================================================================== 2023-10-01T14:53:07.913 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T14:53:07.913 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T14:53:07.913 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T14:53:07.913 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T14:53:07.923 INFO:teuthology.orchestra.run.smithi031.stderr:bash: ntpq: command not found 2023-10-01T14:53:07.927 INFO:teuthology.orchestra.run.smithi031.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-01T14:53:07.927 INFO:teuthology.orchestra.run.smithi031.stdout:=============================================================================== 2023-10-01T14:53:07.927 INFO:teuthology.orchestra.run.smithi031.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T14:53:07.927 INFO:teuthology.orchestra.run.smithi031.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T14:53:07.927 INFO:teuthology.orchestra.run.smithi031.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T14:53:07.927 INFO:teuthology.orchestra.run.smithi031.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T14:53:07.928 INFO:teuthology.run_tasks:Running task pexec... 2023-10-01T14:53:07.938 INFO:teuthology.task.pexec:Executing custom commands... 2023-10-01T14:53:07.938 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi031.front.sepia.ceph.com 2023-10-01T14:53:07.938 DEBUG:teuthology.orchestra.run.smithi031:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-01T14:53:07.939 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi099.front.sepia.ceph.com 2023-10-01T14:53:07.939 DEBUG:teuthology.orchestra.run.smithi099:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-01T14:53:07.939 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi135.front.sepia.ceph.com 2023-10-01T14:53:07.939 DEBUG:teuthology.orchestra.run.smithi135:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-01T14:53:08.818 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:02:22 ago on Sun 01 Oct 2023 02:50:46 PM UTC. 2023-10-01T14:53:08.836 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:02:22 ago on Sun 01 Oct 2023 02:50:46 PM UTC. 2023-10-01T14:53:08.988 INFO:teuthology.orchestra.run.smithi031.stdout:Last metadata expiration check: 0:01:43 ago on Sun 01 Oct 2023 02:51:25 PM UTC. 2023-10-01T14:53:10.830 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-10-01T14:53:10.831 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-10-01T14:53:10.831 INFO:teuthology.orchestra.run.smithi135.stdout: Package Architecture Version Repository Size 2023-10-01T14:53:10.831 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-10-01T14:53:10.831 INFO:teuthology.orchestra.run.smithi135.stdout:Resetting modules: 2023-10-01T14:53:10.831 INFO:teuthology.orchestra.run.smithi135.stdout: container-tools 2023-10-01T14:53:10.831 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:53:10.831 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-10-01T14:53:10.832 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-10-01T14:53:10.832 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:53:10.857 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-10-01T14:53:10.858 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-10-01T14:53:10.858 INFO:teuthology.orchestra.run.smithi099.stdout: Package Architecture Version Repository Size 2023-10-01T14:53:10.858 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-10-01T14:53:10.858 INFO:teuthology.orchestra.run.smithi099.stdout:Resetting modules: 2023-10-01T14:53:10.858 INFO:teuthology.orchestra.run.smithi099.stdout: container-tools 2023-10-01T14:53:10.859 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-10-01T14:53:10.859 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-10-01T14:53:10.859 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-10-01T14:53:10.859 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-10-01T14:53:11.091 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-10-01T14:53:11.109 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-10-01T14:53:11.240 INFO:teuthology.orchestra.run.smithi031.stdout:Dependencies resolved. 2023-10-01T14:53:11.242 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2023-10-01T14:53:11.242 INFO:teuthology.orchestra.run.smithi031.stdout: Package Architecture Version Repository Size 2023-10-01T14:53:11.242 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2023-10-01T14:53:11.242 INFO:teuthology.orchestra.run.smithi031.stdout:Resetting modules: 2023-10-01T14:53:11.242 INFO:teuthology.orchestra.run.smithi031.stdout: container-tools 2023-10-01T14:53:11.242 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:53:11.242 INFO:teuthology.orchestra.run.smithi031.stdout:Transaction Summary 2023-10-01T14:53:11.242 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2023-10-01T14:53:11.242 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:53:11.557 INFO:teuthology.orchestra.run.smithi031.stdout:Complete! 2023-10-01T14:53:12.060 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:02:25 ago on Sun 01 Oct 2023 02:50:46 PM UTC. 2023-10-01T14:53:12.081 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:02:25 ago on Sun 01 Oct 2023 02:50:46 PM UTC. 2023-10-01T14:53:12.638 INFO:teuthology.orchestra.run.smithi031.stdout:Last metadata expiration check: 0:01:47 ago on Sun 01 Oct 2023 02:51:25 PM UTC. 2023-10-01T14:53:14.060 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-10-01T14:53:14.093 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2023-10-01T14:53:14.217 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-10-01T14:53:14.217 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2023-10-01T14:53:14.217 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-10-01T14:53:14.217 INFO:teuthology.orchestra.run.smithi099.stdout:Installing group/module packages: 2023-10-01T14:53:14.217 INFO:teuthology.orchestra.run.smithi099.stdout: aardvark-dns x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 1.0 M 2023-10-01T14:53:14.217 INFO:teuthology.orchestra.run.smithi099.stdout: buildah x86_64 1:1.31.3-1.module_el8+664+4072b3ae appstream 8.8 M 2023-10-01T14:53:14.217 INFO:teuthology.orchestra.run.smithi099.stdout: cockpit-podman noarch 75-1.module_el8+664+4072b3ae appstream 739 k 2023-10-01T14:53:14.217 INFO:teuthology.orchestra.run.smithi099.stdout: crun x86_64 1.8.7-1.module_el8+661+d1afb926 appstream 239 k 2023-10-01T14:53:14.218 INFO:teuthology.orchestra.run.smithi099.stdout: netavark x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 3.7 M 2023-10-01T14:53:14.218 INFO:teuthology.orchestra.run.smithi099.stdout: python3-podman noarch 4.6.0-1.module_el8+582+a2558ae2 appstream 177 k 2023-10-01T14:53:14.218 INFO:teuthology.orchestra.run.smithi099.stdout: skopeo x86_64 2:1.13.3-1.module_el8+664+4072b3ae appstream 8.1 M 2023-10-01T14:53:14.218 INFO:teuthology.orchestra.run.smithi099.stdout: toolbox x86_64 0.0.99.4-5.module_el8+649+e2deac1d appstream 2.5 M 2023-10-01T14:53:14.218 INFO:teuthology.orchestra.run.smithi099.stdout: udica noarch 0.2.6-20.module_el8+487+8e42a277 appstream 53 k 2023-10-01T14:53:14.218 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2023-10-01T14:53:14.218 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2023-10-01T14:53:14.218 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2023-10-01T14:53:14.218 INFO:teuthology.orchestra.run.smithi099.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2023-10-01T14:53:14.218 INFO:teuthology.orchestra.run.smithi099.stdout:Installing weak dependencies: 2023-10-01T14:53:14.218 INFO:teuthology.orchestra.run.smithi099.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2023-10-01T14:53:14.218 INFO:teuthology.orchestra.run.smithi099.stdout:Installing module profiles: 2023-10-01T14:53:14.218 INFO:teuthology.orchestra.run.smithi099.stdout: container-tools/common 2023-10-01T14:53:14.219 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-10-01T14:53:14.219 INFO:teuthology.orchestra.run.smithi099.stdout:Enabling module streams: 2023-10-01T14:53:14.219 INFO:teuthology.orchestra.run.smithi099.stdout: container-tools rhel8 2023-10-01T14:53:14.219 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-10-01T14:53:14.219 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-10-01T14:53:14.219 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-10-01T14:53:14.219 INFO:teuthology.orchestra.run.smithi099.stdout:Install 13 Packages 2023-10-01T14:53:14.219 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-10-01T14:53:14.220 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 26 M 2023-10-01T14:53:14.220 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 87 M 2023-10-01T14:53:14.220 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-10-01T14:53:14.247 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-10-01T14:53:14.247 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2023-10-01T14:53:14.247 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-10-01T14:53:14.248 INFO:teuthology.orchestra.run.smithi135.stdout:Installing group/module packages: 2023-10-01T14:53:14.248 INFO:teuthology.orchestra.run.smithi135.stdout: aardvark-dns x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 1.0 M 2023-10-01T14:53:14.248 INFO:teuthology.orchestra.run.smithi135.stdout: buildah x86_64 1:1.31.3-1.module_el8+664+4072b3ae appstream 8.8 M 2023-10-01T14:53:14.248 INFO:teuthology.orchestra.run.smithi135.stdout: cockpit-podman noarch 75-1.module_el8+664+4072b3ae appstream 739 k 2023-10-01T14:53:14.248 INFO:teuthology.orchestra.run.smithi135.stdout: crun x86_64 1.8.7-1.module_el8+661+d1afb926 appstream 239 k 2023-10-01T14:53:14.248 INFO:teuthology.orchestra.run.smithi135.stdout: netavark x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 3.7 M 2023-10-01T14:53:14.248 INFO:teuthology.orchestra.run.smithi135.stdout: python3-podman noarch 4.6.0-1.module_el8+582+a2558ae2 appstream 177 k 2023-10-01T14:53:14.248 INFO:teuthology.orchestra.run.smithi135.stdout: skopeo x86_64 2:1.13.3-1.module_el8+664+4072b3ae appstream 8.1 M 2023-10-01T14:53:14.248 INFO:teuthology.orchestra.run.smithi135.stdout: toolbox x86_64 0.0.99.4-5.module_el8+649+e2deac1d appstream 2.5 M 2023-10-01T14:53:14.248 INFO:teuthology.orchestra.run.smithi135.stdout: udica noarch 0.2.6-20.module_el8+487+8e42a277 appstream 53 k 2023-10-01T14:53:14.248 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2023-10-01T14:53:14.248 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2023-10-01T14:53:14.248 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2023-10-01T14:53:14.249 INFO:teuthology.orchestra.run.smithi135.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2023-10-01T14:53:14.249 INFO:teuthology.orchestra.run.smithi135.stdout:Installing weak dependencies: 2023-10-01T14:53:14.249 INFO:teuthology.orchestra.run.smithi135.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2023-10-01T14:53:14.249 INFO:teuthology.orchestra.run.smithi135.stdout:Installing module profiles: 2023-10-01T14:53:14.249 INFO:teuthology.orchestra.run.smithi135.stdout: container-tools/common 2023-10-01T14:53:14.249 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:53:14.249 INFO:teuthology.orchestra.run.smithi135.stdout:Enabling module streams: 2023-10-01T14:53:14.249 INFO:teuthology.orchestra.run.smithi135.stdout: container-tools rhel8 2023-10-01T14:53:14.249 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:53:14.249 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2023-10-01T14:53:14.249 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2023-10-01T14:53:14.249 INFO:teuthology.orchestra.run.smithi135.stdout:Install 13 Packages 2023-10-01T14:53:14.249 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:53:14.250 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 26 M 2023-10-01T14:53:14.250 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 87 M 2023-10-01T14:53:14.250 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2023-10-01T14:53:14.755 INFO:teuthology.orchestra.run.smithi099.stdout:(1/13): cockpit-podman-75-1.module_el8+664+4072 1.5 MB/s | 739 kB 00:00 2023-10-01T14:53:14.805 INFO:teuthology.orchestra.run.smithi099.stdout:(2/13): aardvark-dns-1.7.0-1.module_el8+487+8e4 1.8 MB/s | 1.0 MB 00:00 2023-10-01T14:53:14.839 INFO:teuthology.orchestra.run.smithi099.stdout:(3/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 454 kB/s | 38 kB 00:00 2023-10-01T14:53:14.862 INFO:teuthology.orchestra.run.smithi031.stdout:Dependencies resolved. 2023-10-01T14:53:14.897 INFO:teuthology.orchestra.run.smithi099.stdout:(4/13): crun-1.8.7-1.module_el8+661+d1afb926.x8 2.5 MB/s | 239 kB 00:00 2023-10-01T14:53:14.951 INFO:teuthology.orchestra.run.smithi135.stdout:(1/13): cockpit-podman-75-1.module_el8+664+4072 1.4 MB/s | 739 kB 00:00 2023-10-01T14:53:14.989 INFO:teuthology.orchestra.run.smithi099.stdout:(5/13): python3-podman-4.6.0-1.module_el8+582+a 1.9 MB/s | 177 kB 00:00 2023-10-01T14:53:14.993 INFO:teuthology.orchestra.run.smithi135.stdout:(2/13): aardvark-dns-1.7.0-1.module_el8+487+8e4 1.8 MB/s | 1.0 MB 00:00 2023-10-01T14:53:15.036 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2023-10-01T14:53:15.036 INFO:teuthology.orchestra.run.smithi031.stdout: Package Arch Version Repository Size 2023-10-01T14:53:15.036 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2023-10-01T14:53:15.036 INFO:teuthology.orchestra.run.smithi031.stdout:Installing group/module packages: 2023-10-01T14:53:15.036 INFO:teuthology.orchestra.run.smithi031.stdout: aardvark-dns x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 1.0 M 2023-10-01T14:53:15.036 INFO:teuthology.orchestra.run.smithi031.stdout: buildah x86_64 1:1.31.3-1.module_el8+664+4072b3ae appstream 8.8 M 2023-10-01T14:53:15.036 INFO:teuthology.orchestra.run.smithi031.stdout: cockpit-podman noarch 75-1.module_el8+664+4072b3ae appstream 739 k 2023-10-01T14:53:15.036 INFO:teuthology.orchestra.run.smithi031.stdout: crun x86_64 1.8.7-1.module_el8+661+d1afb926 appstream 239 k 2023-10-01T14:53:15.036 INFO:teuthology.orchestra.run.smithi031.stdout: netavark x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 3.7 M 2023-10-01T14:53:15.036 INFO:teuthology.orchestra.run.smithi031.stdout: python3-podman noarch 4.6.0-1.module_el8+582+a2558ae2 appstream 177 k 2023-10-01T14:53:15.037 INFO:teuthology.orchestra.run.smithi031.stdout: skopeo x86_64 2:1.13.3-1.module_el8+664+4072b3ae appstream 8.1 M 2023-10-01T14:53:15.037 INFO:teuthology.orchestra.run.smithi031.stdout: toolbox x86_64 0.0.99.4-5.module_el8+649+e2deac1d appstream 2.5 M 2023-10-01T14:53:15.037 INFO:teuthology.orchestra.run.smithi031.stdout: udica noarch 0.2.6-20.module_el8+487+8e42a277 appstream 53 k 2023-10-01T14:53:15.037 INFO:teuthology.orchestra.run.smithi031.stdout:Installing dependencies: 2023-10-01T14:53:15.037 INFO:teuthology.orchestra.run.smithi031.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2023-10-01T14:53:15.037 INFO:teuthology.orchestra.run.smithi031.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2023-10-01T14:53:15.037 INFO:teuthology.orchestra.run.smithi031.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2023-10-01T14:53:15.037 INFO:teuthology.orchestra.run.smithi031.stdout:Installing weak dependencies: 2023-10-01T14:53:15.037 INFO:teuthology.orchestra.run.smithi031.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2023-10-01T14:53:15.037 INFO:teuthology.orchestra.run.smithi031.stdout:Installing module profiles: 2023-10-01T14:53:15.037 INFO:teuthology.orchestra.run.smithi031.stdout: container-tools/common 2023-10-01T14:53:15.037 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:53:15.037 INFO:teuthology.orchestra.run.smithi031.stdout:Enabling module streams: 2023-10-01T14:53:15.038 INFO:teuthology.orchestra.run.smithi031.stdout: container-tools rhel8 2023-10-01T14:53:15.038 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:53:15.038 INFO:teuthology.orchestra.run.smithi031.stdout:Transaction Summary 2023-10-01T14:53:15.038 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2023-10-01T14:53:15.038 INFO:teuthology.orchestra.run.smithi031.stdout:Install 13 Packages 2023-10-01T14:53:15.038 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:53:15.038 INFO:teuthology.orchestra.run.smithi031.stdout:Total download size: 26 M 2023-10-01T14:53:15.039 INFO:teuthology.orchestra.run.smithi031.stdout:Installed size: 87 M 2023-10-01T14:53:15.039 INFO:teuthology.orchestra.run.smithi031.stdout:Downloading Packages: 2023-10-01T14:53:15.043 INFO:teuthology.orchestra.run.smithi135.stdout:(3/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 412 kB/s | 38 kB 00:00 2023-10-01T14:53:15.073 INFO:teuthology.orchestra.run.smithi099.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 297 kB/s | 25 kB 00:00 2023-10-01T14:53:15.118 INFO:teuthology.orchestra.run.smithi135.stdout:(4/13): crun-1.8.7-1.module_el8+661+d1afb926.x8 1.9 MB/s | 239 kB 00:00 2023-10-01T14:53:15.173 INFO:teuthology.orchestra.run.smithi099.stdout:(7/13): python3-pyxdg-0.25-16.el8.noarch.rpm 940 kB/s | 94 kB 00:00 2023-10-01T14:53:15.219 INFO:teuthology.orchestra.run.smithi135.stdout:(5/13): python3-podman-4.6.0-1.module_el8+582+a 1.7 MB/s | 177 kB 00:00 2023-10-01T14:53:15.311 INFO:teuthology.orchestra.run.smithi135.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 270 kB/s | 25 kB 00:00 2023-10-01T14:53:15.403 INFO:teuthology.orchestra.run.smithi135.stdout:(7/13): python3-pyxdg-0.25-16.el8.noarch.rpm 1.0 MB/s | 94 kB 00:00 2023-10-01T14:53:15.565 INFO:teuthology.orchestra.run.smithi099.stdout:(8/13): netavark-1.7.0-1.module_el8+487+8e42a27 5.2 MB/s | 3.7 MB 00:00 2023-10-01T14:53:15.725 INFO:teuthology.orchestra.run.smithi031.stdout:(1/13): cockpit-podman-75-1.module_el8+664+4072 1.4 MB/s | 739 kB 00:00 2023-10-01T14:53:15.737 INFO:teuthology.orchestra.run.smithi135.stdout:(8/13): netavark-1.7.0-1.module_el8+487+8e42a27 5.4 MB/s | 3.7 MB 00:00 2023-10-01T14:53:15.800 INFO:teuthology.orchestra.run.smithi031.stdout:(2/13): aardvark-dns-1.7.0-1.module_el8+487+8e4 1.7 MB/s | 1.0 MB 00:00 2023-10-01T14:53:15.825 INFO:teuthology.orchestra.run.smithi031.stdout:(3/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 379 kB/s | 38 kB 00:00 2023-10-01T14:53:15.892 INFO:teuthology.orchestra.run.smithi031.stdout:(4/13): crun-1.8.7-1.module_el8+661+d1afb926.x8 2.5 MB/s | 239 kB 00:00 2023-10-01T14:53:15.992 INFO:teuthology.orchestra.run.smithi031.stdout:(5/13): python3-podman-4.6.0-1.module_el8+582+a 1.7 MB/s | 177 kB 00:00 2023-10-01T14:53:16.024 INFO:teuthology.orchestra.run.smithi099.stdout:(9/13): buildah-1.31.3-1.module_el8+664+4072b3a 5.0 MB/s | 8.8 MB 00:01 2023-10-01T14:53:16.074 INFO:teuthology.orchestra.run.smithi099.stdout:(10/13): toolbox-0.0.99.4-5.module_el8+649+e2de 5.0 MB/s | 2.5 MB 00:00 2023-10-01T14:53:16.101 INFO:teuthology.orchestra.run.smithi031.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 229 kB/s | 25 kB 00:00 2023-10-01T14:53:16.108 INFO:teuthology.orchestra.run.smithi099.stdout:(11/13): udica-0.2.6-20.module_el8+487+8e42a277 630 kB/s | 53 kB 00:00 2023-10-01T14:53:16.158 INFO:teuthology.orchestra.run.smithi099.stdout:(12/13): yajl-2.1.0-12.el8.x86_64.rpm 490 kB/s | 41 kB 00:00 2023-10-01T14:53:16.184 INFO:teuthology.orchestra.run.smithi135.stdout:(9/13): buildah-1.31.3-1.module_el8+664+4072b3a 5.1 MB/s | 8.8 MB 00:01 2023-10-01T14:53:16.184 INFO:teuthology.orchestra.run.smithi031.stdout:(7/13): python3-pyxdg-0.25-16.el8.noarch.rpm 1.1 MB/s | 94 kB 00:00 2023-10-01T14:53:16.383 INFO:teuthology.orchestra.run.smithi099.stdout:(13/13): skopeo-1.13.3-1.module_el8+664+4072b3a 6.7 MB/s | 8.1 MB 00:01 2023-10-01T14:53:16.383 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-10-01T14:53:16.384 INFO:teuthology.orchestra.run.smithi099.stdout:Total 12 MB/s | 26 MB 00:02 2023-10-01T14:53:16.457 INFO:teuthology.orchestra.run.smithi135.stdout:(10/13): toolbox-0.0.99.4-5.module_el8+649+e2de 3.5 MB/s | 2.5 MB 00:00 2023-10-01T14:53:16.527 INFO:teuthology.orchestra.run.smithi031.stdout:(8/13): netavark-1.7.0-1.module_el8+487+8e42a27 5.3 MB/s | 3.7 MB 00:00 2023-10-01T14:53:16.598 INFO:teuthology.orchestra.run.smithi135.stdout:(11/13): udica-0.2.6-20.module_el8+487+8e42a277 127 kB/s | 53 kB 00:00 2023-10-01T14:53:16.630 INFO:teuthology.orchestra.run.smithi135.stdout:(12/13): yajl-2.1.0-12.el8.x86_64.rpm 237 kB/s | 41 kB 00:00 2023-10-01T14:53:16.648 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-10-01T14:53:16.669 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-10-01T14:53:16.669 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-10-01T14:53:16.796 INFO:teuthology.orchestra.run.smithi135.stdout:(13/13): skopeo-1.13.3-1.module_el8+664+4072b3a 5.8 MB/s | 8.1 MB 00:01 2023-10-01T14:53:16.797 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2023-10-01T14:53:16.797 INFO:teuthology.orchestra.run.smithi135.stdout:Total 10 MB/s | 26 MB 00:02 2023-10-01T14:53:16.935 INFO:teuthology.orchestra.run.smithi031.stdout:(9/13): buildah-1.31.3-1.module_el8+664+4072b3a 5.2 MB/s | 8.8 MB 00:01 2023-10-01T14:53:17.020 INFO:teuthology.orchestra.run.smithi031.stdout:(10/13): toolbox-0.0.99.4-5.module_el8+649+e2de 5.1 MB/s | 2.5 MB 00:00 2023-10-01T14:53:17.052 INFO:teuthology.orchestra.run.smithi031.stdout:(11/13): udica-0.2.6-20.module_el8+487+8e42a277 451 kB/s | 53 kB 00:00 2023-10-01T14:53:17.064 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2023-10-01T14:53:17.085 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2023-10-01T14:53:17.085 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2023-10-01T14:53:17.102 INFO:teuthology.orchestra.run.smithi031.stdout:(12/13): yajl-2.1.0-12.el8.x86_64.rpm 498 kB/s | 41 kB 00:00 2023-10-01T14:53:17.159 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-10-01T14:53:17.163 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-10-01T14:53:17.339 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2023-10-01T14:53:17.343 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2023-10-01T14:53:17.419 INFO:teuthology.orchestra.run.smithi031.stdout:(13/13): skopeo-1.13.3-1.module_el8+664+4072b3a 6.6 MB/s | 8.1 MB 00:01 2023-10-01T14:53:17.420 INFO:teuthology.orchestra.run.smithi031.stdout:-------------------------------------------------------------------------------- 2023-10-01T14:53:17.420 INFO:teuthology.orchestra.run.smithi031.stdout:Total 11 MB/s | 26 MB 00:02 2023-10-01T14:53:17.707 INFO:teuthology.orchestra.run.smithi031.stdout:Running transaction check 2023-10-01T14:53:17.731 INFO:teuthology.orchestra.run.smithi031.stdout:Transaction check succeeded. 2023-10-01T14:53:17.731 INFO:teuthology.orchestra.run.smithi031.stdout:Running transaction test 2023-10-01T14:53:17.927 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-10-01T14:53:18.028 INFO:teuthology.orchestra.run.smithi031.stdout:Transaction test succeeded. 2023-10-01T14:53:18.032 INFO:teuthology.orchestra.run.smithi031.stdout:Running transaction 2023-10-01T14:53:18.069 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2023-10-01T14:53:18.131 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2023-10-01T14:53:18.293 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/13 2023-10-01T14:53:18.323 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2023-10-01T14:53:18.501 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/13 2023-10-01T14:53:18.531 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/13 2023-10-01T14:53:18.732 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/13 2023-10-01T14:53:18.735 INFO:teuthology.orchestra.run.smithi031.stdout: Preparing : 1/1 2023-10-01T14:53:18.754 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-10-01T14:53:18.980 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2023-10-01T14:53:19.004 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-10-01T14:53:19.172 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/13 2023-10-01T14:53:19.321 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 5/13 2023-10-01T14:53:19.365 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/13 2023-10-01T14:53:19.570 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 5/13 2023-10-01T14:53:19.639 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-10-01T14:53:20.229 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-10-01T14:53:20.261 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 5/13 2023-10-01T14:53:20.472 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 7/13 2023-10-01T14:53:20.525 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-10-01T14:53:20.686 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : crun-1.8.7-1.module_el8+661+d1afb926.x86_64 8/13 2023-10-01T14:53:20.770 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 7/13 2023-10-01T14:53:20.938 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 9/13 2023-10-01T14:53:21.009 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : crun-1.8.7-1.module_el8+661+d1afb926.x86_64 8/13 2023-10-01T14:53:21.288 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 9/13 2023-10-01T14:53:21.393 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : udica-0.2.6-20.module_el8+487+8e42a277.noarch 10/13 2023-10-01T14:53:21.619 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-10-01T14:53:21.713 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : udica-0.2.6-20.module_el8+487+8e42a277.noarch 10/13 2023-10-01T14:53:21.862 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 7/13 2023-10-01T14:53:22.070 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : crun-1.8.7-1.module_el8+661+d1afb926.x86_64 8/13 2023-10-01T14:53:22.268 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-10-01T14:53:22.344 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 9/13 2023-10-01T14:53:22.504 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 12/13 2023-10-01T14:53:22.708 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 13/13 2023-10-01T14:53:23.063 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-10-01T14:53:23.063 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : udica-0.2.6-20.module_el8+487+8e42a277.noarch 10/13 2023-10-01T14:53:23.100 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 12/13 2023-10-01T14:53:23.314 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 13/13 2023-10-01T14:53:23.314 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 1/13 2023-10-01T14:53:23.315 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 2/13 2023-10-01T14:53:23.315 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 3/13 2023-10-01T14:53:23.315 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-10-01T14:53:23.315 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : crun-1.8.7-1.module_el8+661+d1afb926.x86_64 5/13 2023-10-01T14:53:23.315 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-10-01T14:53:23.316 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 7/13 2023-10-01T14:53:23.316 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2023-10-01T14:53:23.316 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2023-10-01T14:53:23.316 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 10/13 2023-10-01T14:53:23.317 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-10-01T14:53:23.317 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : udica-0.2.6-20.module_el8+487+8e42a277.noarch 12/13 2023-10-01T14:53:23.333 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 13/13 2023-10-01T14:53:23.728 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-10-01T14:53:23.774 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 13/13 2023-10-01T14:53:23.774 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 1/13 2023-10-01T14:53:23.774 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 2/13 2023-10-01T14:53:23.774 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 3/13 2023-10-01T14:53:23.774 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-10-01T14:53:23.774 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : crun-1.8.7-1.module_el8+661+d1afb926.x86_64 5/13 2023-10-01T14:53:23.774 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-10-01T14:53:23.774 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 7/13 2023-10-01T14:53:23.774 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2023-10-01T14:53:23.775 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2023-10-01T14:53:23.775 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 10/13 2023-10-01T14:53:23.775 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-10-01T14:53:23.775 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : udica-0.2.6-20.module_el8+487+8e42a277.noarch 12/13 2023-10-01T14:53:23.887 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2023-10-01T14:53:24.125 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-10-01T14:53:24.126 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-10-01T14:53:24.126 INFO:teuthology.orchestra.run.smithi099.stdout: aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-10-01T14:53:24.126 INFO:teuthology.orchestra.run.smithi099.stdout: buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 2023-10-01T14:53:24.126 INFO:teuthology.orchestra.run.smithi099.stdout: cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 2023-10-01T14:53:24.126 INFO:teuthology.orchestra.run.smithi099.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2023-10-01T14:53:24.126 INFO:teuthology.orchestra.run.smithi099.stdout: crun-1.8.7-1.module_el8+661+d1afb926.x86_64 2023-10-01T14:53:24.126 INFO:teuthology.orchestra.run.smithi099.stdout: netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-10-01T14:53:24.126 INFO:teuthology.orchestra.run.smithi099.stdout: python3-podman-4.6.0-1.module_el8+582+a2558ae2.noarch 2023-10-01T14:53:24.126 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-10-01T14:53:24.127 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-10-01T14:53:24.127 INFO:teuthology.orchestra.run.smithi099.stdout: skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 2023-10-01T14:53:24.127 INFO:teuthology.orchestra.run.smithi099.stdout: toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 2023-10-01T14:53:24.127 INFO:teuthology.orchestra.run.smithi099.stdout: udica-0.2.6-20.module_el8+487+8e42a277.noarch 2023-10-01T14:53:24.127 INFO:teuthology.orchestra.run.smithi099.stdout: yajl-2.1.0-12.el8.x86_64 2023-10-01T14:53:24.127 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-10-01T14:53:24.127 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-10-01T14:53:24.128 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 12/13 2023-10-01T14:53:24.136 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 13/13 2023-10-01T14:53:24.206 DEBUG:teuthology.parallel:result is None 2023-10-01T14:53:24.535 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2023-10-01T14:53:24.535 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:53:24.535 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2023-10-01T14:53:24.535 INFO:teuthology.orchestra.run.smithi135.stdout: aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-10-01T14:53:24.535 INFO:teuthology.orchestra.run.smithi135.stdout: buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 2023-10-01T14:53:24.536 INFO:teuthology.orchestra.run.smithi135.stdout: cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 2023-10-01T14:53:24.536 INFO:teuthology.orchestra.run.smithi135.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2023-10-01T14:53:24.536 INFO:teuthology.orchestra.run.smithi135.stdout: crun-1.8.7-1.module_el8+661+d1afb926.x86_64 2023-10-01T14:53:24.536 INFO:teuthology.orchestra.run.smithi135.stdout: netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-10-01T14:53:24.536 INFO:teuthology.orchestra.run.smithi135.stdout: python3-podman-4.6.0-1.module_el8+582+a2558ae2.noarch 2023-10-01T14:53:24.536 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-10-01T14:53:24.536 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-10-01T14:53:24.536 INFO:teuthology.orchestra.run.smithi135.stdout: skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 2023-10-01T14:53:24.536 INFO:teuthology.orchestra.run.smithi135.stdout: toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 2023-10-01T14:53:24.536 INFO:teuthology.orchestra.run.smithi135.stdout: udica-0.2.6-20.module_el8+487+8e42a277.noarch 2023-10-01T14:53:24.536 INFO:teuthology.orchestra.run.smithi135.stdout: yajl-2.1.0-12.el8.x86_64 2023-10-01T14:53:24.537 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:53:24.537 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2023-10-01T14:53:24.620 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 13/13 2023-10-01T14:53:24.620 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 1/13 2023-10-01T14:53:24.620 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 2/13 2023-10-01T14:53:24.620 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 3/13 2023-10-01T14:53:24.620 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-10-01T14:53:24.620 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : crun-1.8.7-1.module_el8+661+d1afb926.x86_64 5/13 2023-10-01T14:53:24.620 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-10-01T14:53:24.620 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 7/13 2023-10-01T14:53:24.620 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2023-10-01T14:53:24.620 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2023-10-01T14:53:24.620 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 10/13 2023-10-01T14:53:24.620 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-10-01T14:53:24.621 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : udica-0.2.6-20.module_el8+487+8e42a277.noarch 12/13 2023-10-01T14:53:24.907 DEBUG:teuthology.parallel:result is None 2023-10-01T14:53:25.423 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2023-10-01T14:53:25.423 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:53:25.423 INFO:teuthology.orchestra.run.smithi031.stdout:Installed: 2023-10-01T14:53:25.423 INFO:teuthology.orchestra.run.smithi031.stdout: aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-10-01T14:53:25.424 INFO:teuthology.orchestra.run.smithi031.stdout: buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 2023-10-01T14:53:25.424 INFO:teuthology.orchestra.run.smithi031.stdout: cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 2023-10-01T14:53:25.424 INFO:teuthology.orchestra.run.smithi031.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2023-10-01T14:53:25.424 INFO:teuthology.orchestra.run.smithi031.stdout: crun-1.8.7-1.module_el8+661+d1afb926.x86_64 2023-10-01T14:53:25.424 INFO:teuthology.orchestra.run.smithi031.stdout: netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-10-01T14:53:25.424 INFO:teuthology.orchestra.run.smithi031.stdout: python3-podman-4.6.0-1.module_el8+582+a2558ae2.noarch 2023-10-01T14:53:25.424 INFO:teuthology.orchestra.run.smithi031.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-10-01T14:53:25.424 INFO:teuthology.orchestra.run.smithi031.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-10-01T14:53:25.424 INFO:teuthology.orchestra.run.smithi031.stdout: skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 2023-10-01T14:53:25.424 INFO:teuthology.orchestra.run.smithi031.stdout: toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 2023-10-01T14:53:25.424 INFO:teuthology.orchestra.run.smithi031.stdout: udica-0.2.6-20.module_el8+487+8e42a277.noarch 2023-10-01T14:53:25.424 INFO:teuthology.orchestra.run.smithi031.stdout: yajl-2.1.0-12.el8.x86_64 2023-10-01T14:53:25.425 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:53:25.425 INFO:teuthology.orchestra.run.smithi031.stdout:Complete! 2023-10-01T14:53:25.786 DEBUG:teuthology.parallel:result is None 2023-10-01T14:53:25.786 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-01T14:53:25.856 INFO:tasks.cephadm:Config: {'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bcea8c48517ff154f392584022efa0718f51c9f2'} 2023-10-01T14:53:25.856 INFO:tasks.cephadm:Cluster image is quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 2023-10-01T14:53:25.856 INFO:tasks.cephadm:Cluster fsid is 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:53:25.856 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-01T14:53:25.857 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.31', 'mon.b': '172.21.15.99', 'mon.c': '172.21.15.135'} 2023-10-01T14:53:25.857 INFO:tasks.cephadm:First mon is mon.a on smithi031 2023-10-01T14:53:25.857 INFO:tasks.cephadm:First mgr is x 2023-10-01T14:53:25.857 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-01T14:53:25.857 DEBUG:teuthology.orchestra.run.smithi031:> sudo hostname $(hostname -s) 2023-10-01T14:53:25.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo hostname $(hostname -s) 2023-10-01T14:53:25.927 DEBUG:teuthology.orchestra.run.smithi135:> sudo hostname $(hostname -s) 2023-10-01T14:53:25.959 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra 2023-10-01T14:53:25.959 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=bcea8c48517ff154f392584022efa0718f51c9f2 2023-10-01T14:53:26.178 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': 'bcea8c48517ff154f392584022efa0718f51c9f2', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-new-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/72782/', 'root_build_cause': 'SCMTRIGGER', 'version': '18.0.0-5817-gbcea8c48', 'node_name': '172.21.4.65+confusa03', 'job_name': 'ceph-dev-new-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic', 'package_manager_version': '18.0.0-5817.gbcea8c48'}, 'url': 'https://2.chacra.ceph.com/r/ceph/wip-barakda-cephadm-nvmeof/bcea8c48517ff154f392584022efa0718f51c9f2/centos/8/flavors/default/', 'distro_codename': None, 'modified': '2023-09-25 21:06:38.915379', 'distro_version': '8', 'project': 'ceph', 'flavor': 'default', 'ref': 'wip-barakda-cephadm-nvmeof', 'chacra_url': 'https://2.chacra.ceph.com/repos/ceph/wip-barakda-cephadm-nvmeof/bcea8c48517ff154f392584022efa0718f51c9f2/centos/8/flavors/default/', 'archs': ['x86_64', 'source', 'arm64'], 'distro': 'centos'}] 2023-10-01T14:53:26.396 INFO:tasks.util.chacra:got chacra host 2.chacra.ceph.com, ref wip-barakda-cephadm-nvmeof, sha1 bcea8c48517ff154f392584022efa0718f51c9f2 from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F8%2Fx86_64&flavor=default&sha1=bcea8c48517ff154f392584022efa0718f51c9f2 2023-10-01T14:53:26.397 INFO:tasks.cephadm:Discovered cachra url: https://2.chacra.ceph.com/binaries/ceph/wip-barakda-cephadm-nvmeof/bcea8c48517ff154f392584022efa0718f51c9f2/centos/8/x86_64/flavors/default/cephadm 2023-10-01T14:53:26.397 DEBUG:teuthology.orchestra.run.smithi031:> curl --silent -L https://2.chacra.ceph.com/binaries/ceph/wip-barakda-cephadm-nvmeof/bcea8c48517ff154f392584022efa0718f51c9f2/centos/8/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-01T14:53:26.823 INFO:teuthology.orchestra.run.smithi031.stdout:-rw-rw-r--. 1 ubuntu ubuntu 705688 Oct 1 14:53 /home/ubuntu/cephtest/cephadm 2023-10-01T14:53:26.823 DEBUG:teuthology.orchestra.run.smithi099:> curl --silent -L https://2.chacra.ceph.com/binaries/ceph/wip-barakda-cephadm-nvmeof/bcea8c48517ff154f392584022efa0718f51c9f2/centos/8/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-01T14:53:27.357 INFO:teuthology.orchestra.run.smithi099.stdout:-rw-rw-r--. 1 ubuntu ubuntu 705688 Oct 1 14:53 /home/ubuntu/cephtest/cephadm 2023-10-01T14:53:27.358 DEBUG:teuthology.orchestra.run.smithi135:> curl --silent -L https://2.chacra.ceph.com/binaries/ceph/wip-barakda-cephadm-nvmeof/bcea8c48517ff154f392584022efa0718f51c9f2/centos/8/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-01T14:53:27.739 INFO:teuthology.orchestra.run.smithi135.stdout:-rw-rw-r--. 1 ubuntu ubuntu 705688 Oct 1 14:53 /home/ubuntu/cephtest/cephadm 2023-10-01T14:53:27.739 DEBUG:teuthology.orchestra.run.smithi031:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-01T14:53:27.763 DEBUG:teuthology.orchestra.run.smithi099:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-01T14:53:27.786 DEBUG:teuthology.orchestra.run.smithi135:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-01T14:53:27.819 INFO:tasks.cephadm:Pulling image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 on all hosts... 2023-10-01T14:53:27.819 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 pull 2023-10-01T14:53:27.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 pull 2023-10-01T14:53:27.828 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 pull 2023-10-01T14:53:29.405 INFO:teuthology.orchestra.run.smithi031.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2... 2023-10-01T14:53:29.484 INFO:teuthology.orchestra.run.smithi099.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2... 2023-10-01T14:53:29.548 INFO:teuthology.orchestra.run.smithi135.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2... 2023-10-01T14:53:54.076 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-10-01T14:53:54.100 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph_version": "ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev)", 2023-10-01T14:53:54.100 INFO:teuthology.orchestra.run.smithi099.stdout: "image_id": "fa922c4449069cae14b17e9ffaef6cf64d1d7080cea061a20dc1d8f318e65863", 2023-10-01T14:53:54.100 INFO:teuthology.orchestra.run.smithi099.stdout: "repo_digests": [ 2023-10-01T14:53:54.100 INFO:teuthology.orchestra.run.smithi099.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:34fad0ea24358428207f7cbc4d6027320c5a86fd3de6f51aaec11e44cfc648d7" 2023-10-01T14:53:54.101 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-10-01T14:53:54.101 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-10-01T14:53:58.710 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2023-10-01T14:53:58.710 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph_version": "ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev)", 2023-10-01T14:53:58.710 INFO:teuthology.orchestra.run.smithi135.stdout: "image_id": "fa922c4449069cae14b17e9ffaef6cf64d1d7080cea061a20dc1d8f318e65863", 2023-10-01T14:53:58.710 INFO:teuthology.orchestra.run.smithi135.stdout: "repo_digests": [ 2023-10-01T14:53:58.710 INFO:teuthology.orchestra.run.smithi135.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:34fad0ea24358428207f7cbc4d6027320c5a86fd3de6f51aaec11e44cfc648d7" 2023-10-01T14:53:58.710 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2023-10-01T14:53:58.710 INFO:teuthology.orchestra.run.smithi135.stdout:} 2023-10-01T14:54:04.977 INFO:teuthology.orchestra.run.smithi031.stdout:{ 2023-10-01T14:54:04.977 INFO:teuthology.orchestra.run.smithi031.stdout: "ceph_version": "ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev)", 2023-10-01T14:54:04.977 INFO:teuthology.orchestra.run.smithi031.stdout: "image_id": "fa922c4449069cae14b17e9ffaef6cf64d1d7080cea061a20dc1d8f318e65863", 2023-10-01T14:54:04.977 INFO:teuthology.orchestra.run.smithi031.stdout: "repo_digests": [ 2023-10-01T14:54:04.977 INFO:teuthology.orchestra.run.smithi031.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:34fad0ea24358428207f7cbc4d6027320c5a86fd3de6f51aaec11e44cfc648d7" 2023-10-01T14:54:04.978 INFO:teuthology.orchestra.run.smithi031.stdout: ] 2023-10-01T14:54:04.978 INFO:teuthology.orchestra.run.smithi031.stdout:} 2023-10-01T14:54:05.004 DEBUG:teuthology.orchestra.run.smithi031:> sudo mkdir -p /etc/ceph 2023-10-01T14:54:05.041 DEBUG:teuthology.orchestra.run.smithi099:> sudo mkdir -p /etc/ceph 2023-10-01T14:54:05.079 DEBUG:teuthology.orchestra.run.smithi135:> sudo mkdir -p /etc/ceph 2023-10-01T14:54:05.116 DEBUG:teuthology.orchestra.run.smithi031:> sudo chmod 777 /etc/ceph 2023-10-01T14:54:05.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 777 /etc/ceph 2023-10-01T14:54:05.181 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod 777 /etc/ceph 2023-10-01T14:54:05.214 INFO:tasks.cephadm:Writing seed config... 2023-10-01T14:54:05.214 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-01T14:54:05.215 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-01T14:54:05.215 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-01T14:54:05.215 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-01T14:54:05.215 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-01T14:54:05.215 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-01T14:54:05.215 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-01T14:54:05.216 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2023-10-01T14:54:05.216 DEBUG:teuthology.orchestra.run.smithi031:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-01T14:54:05.238 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 466018a4-606a-11ee-8db5-212e2dc638e7 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-01T14:54:05.239 DEBUG:teuthology.orchestra.run.smithi031:mon.a> sudo journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-01T14:54:05.282 DEBUG:teuthology.orchestra.run.smithi031:mgr.x> sudo journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-01T14:54:05.315 INFO:journalctl@ceph.mon.a.smithi031.stdout:-- Logs begin at Sun 2023-10-01 14:44:31 UTC. -- 2023-10-01T14:54:05.316 INFO:tasks.cephadm:Bootstrapping... 2023-10-01T14:54:05.317 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 -v bootstrap --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id x --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.31 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-01T14:54:05.349 INFO:journalctl@ceph.mgr.x.smithi031.stdout:-- Logs begin at Sun 2023-10-01 14:44:31 UTC. -- 2023-10-01T14:54:05.506 INFO:teuthology.orchestra.run.smithi031.stdout:-------------------------------------------------------------------------------- 2023-10-01T14:54:05.506 INFO:teuthology.orchestra.run.smithi031.stdout:cephadm ['--image', 'quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2', '-v', 'bootstrap', '--fsid', '466018a4-606a-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'x', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.31', '--skip-admin-label'] 2023-10-01T14:54:05.540 INFO:teuthology.orchestra.run.smithi031.stdout:/bin/podman: stdout 4.6.1 2023-10-01T14:54:05.541 INFO:teuthology.orchestra.run.smithi031.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-10-01T14:54:05.541 INFO:teuthology.orchestra.run.smithi031.stdout:Verifying podman|docker is present... 2023-10-01T14:54:05.572 INFO:teuthology.orchestra.run.smithi031.stdout:/bin/podman: stdout 4.6.1 2023-10-01T14:54:05.572 INFO:teuthology.orchestra.run.smithi031.stdout:Verifying lvm2 is present... 2023-10-01T14:54:05.572 INFO:teuthology.orchestra.run.smithi031.stdout:Verifying time synchronization is in place... 2023-10-01T14:54:05.580 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-01T14:54:05.580 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-01T14:54:05.586 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-01T14:54:05.586 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout inactive 2023-10-01T14:54:05.593 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout enabled 2023-10-01T14:54:05.600 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout active 2023-10-01T14:54:05.600 INFO:teuthology.orchestra.run.smithi031.stdout:Unit chronyd.service is enabled and running 2023-10-01T14:54:05.600 INFO:teuthology.orchestra.run.smithi031.stdout:Repeating the final host check... 2023-10-01T14:54:05.631 INFO:teuthology.orchestra.run.smithi031.stdout:/bin/podman: stdout 4.6.1 2023-10-01T14:54:05.631 INFO:teuthology.orchestra.run.smithi031.stdout:podman (/bin/podman) version 4.6.1 is present 2023-10-01T14:54:05.631 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl is present 2023-10-01T14:54:05.632 INFO:teuthology.orchestra.run.smithi031.stdout:lvcreate is present 2023-10-01T14:54:05.638 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-01T14:54:05.638 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-01T14:54:05.645 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-01T14:54:05.645 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout inactive 2023-10-01T14:54:05.651 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout enabled 2023-10-01T14:54:05.658 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout active 2023-10-01T14:54:05.658 INFO:teuthology.orchestra.run.smithi031.stdout:Unit chronyd.service is enabled and running 2023-10-01T14:54:05.658 INFO:teuthology.orchestra.run.smithi031.stdout:Host looks OK 2023-10-01T14:54:05.658 INFO:teuthology.orchestra.run.smithi031.stdout:Cluster fsid: 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:54:05.659 INFO:teuthology.orchestra.run.smithi031.stdout:Acquiring lock 139937692934944 on /run/cephadm/466018a4-606a-11ee-8db5-212e2dc638e7.lock 2023-10-01T14:54:05.659 INFO:teuthology.orchestra.run.smithi031.stdout:Lock 139937692934944 acquired on /run/cephadm/466018a4-606a-11ee-8db5-212e2dc638e7.lock 2023-10-01T14:54:05.659 INFO:teuthology.orchestra.run.smithi031.stdout:Verifying IP 172.21.15.31 port 3300 ... 2023-10-01T14:54:05.660 INFO:teuthology.orchestra.run.smithi031.stdout:Verifying IP 172.21.15.31 port 6789 ... 2023-10-01T14:54:05.660 INFO:teuthology.orchestra.run.smithi031.stdout:Base mon IP(s) is [172.21.15.31:3300, 172.21.15.31:6789], mon addrv is [v2:172.21.15.31:3300,v1:172.21.15.31:6789] 2023-10-01T14:54:05.664 INFO:teuthology.orchestra.run.smithi031.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.31 metric 100 2023-10-01T14:54:05.664 INFO:teuthology.orchestra.run.smithi031.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.31 metric 100 2023-10-01T14:54:05.667 INFO:teuthology.orchestra.run.smithi031.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-10-01T14:54:05.667 INFO:teuthology.orchestra.run.smithi031.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-10-01T14:54:05.667 INFO:teuthology.orchestra.run.smithi031.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1758sec hoplimit 64 pref medium 2023-10-01T14:54:05.670 INFO:teuthology.orchestra.run.smithi031.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-01T14:54:05.670 INFO:teuthology.orchestra.run.smithi031.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-10-01T14:54:05.670 INFO:teuthology.orchestra.run.smithi031.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-01T14:54:05.670 INFO:teuthology.orchestra.run.smithi031.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2023-10-01T14:54:05.670 INFO:teuthology.orchestra.run.smithi031.stdout:/sbin/ip: stdout inet6 fe80::ae1f:6bff:fea5:caba/64 scope link 2023-10-01T14:54:05.670 INFO:teuthology.orchestra.run.smithi031.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-01T14:54:05.671 INFO:teuthology.orchestra.run.smithi031.stdout:Mon IP `172.21.15.31` is in CIDR network `172.21.0.0/20` 2023-10-01T14:54:05.671 INFO:teuthology.orchestra.run.smithi031.stdout:Mon IP `172.21.15.31` is in CIDR network `172.21.0.0/20` 2023-10-01T14:54:05.671 INFO:teuthology.orchestra.run.smithi031.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-10-01T14:54:05.672 INFO:teuthology.orchestra.run.smithi031.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-01T14:54:05.673 INFO:teuthology.orchestra.run.smithi031.stdout:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2... 2023-10-01T14:54:06.037 INFO:teuthology.orchestra.run.smithi031.stdout:/bin/podman: stdout fa922c4449069cae14b17e9ffaef6cf64d1d7080cea061a20dc1d8f318e65863 2023-10-01T14:54:06.037 INFO:teuthology.orchestra.run.smithi031.stdout:/bin/podman: stderr Trying to pull quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2... 2023-10-01T14:54:06.037 INFO:teuthology.orchestra.run.smithi031.stdout:/bin/podman: stderr Getting image source signatures 2023-10-01T14:54:06.037 INFO:teuthology.orchestra.run.smithi031.stdout:/bin/podman: stderr Copying blob sha256:556fc22cf1e739705d2013195476c145689a6e17c5a9071e0d49fdc6e036642a 2023-10-01T14:54:06.037 INFO:teuthology.orchestra.run.smithi031.stdout:/bin/podman: stderr Copying blob sha256:3f4fd5c3c800c6da9ef94691df203a05a5bbbf32c483d09d620ec06eef42ea2b 2023-10-01T14:54:06.037 INFO:teuthology.orchestra.run.smithi031.stdout:/bin/podman: stderr Copying config sha256:fa922c4449069cae14b17e9ffaef6cf64d1d7080cea061a20dc1d8f318e65863 2023-10-01T14:54:06.037 INFO:teuthology.orchestra.run.smithi031.stdout:/bin/podman: stderr Writing manifest to image destination 2023-10-01T14:54:06.685 INFO:teuthology.orchestra.run.smithi031.stdout:ceph: stdout ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev) 2023-10-01T14:54:06.685 INFO:teuthology.orchestra.run.smithi031.stdout:Ceph version: ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev) 2023-10-01T14:54:06.685 INFO:teuthology.orchestra.run.smithi031.stdout:Extracting ceph user uid/gid from container image... 2023-10-01T14:54:07.246 INFO:teuthology.orchestra.run.smithi031.stdout:stat: stdout 167 167 2023-10-01T14:54:07.246 INFO:teuthology.orchestra.run.smithi031.stdout:Creating initial keys... 2023-10-01T14:54:07.781 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph-authtool: stdout AQAPiBllTuICGhAAxUSxITqMypqmOVZW3TGAGg== 2023-10-01T14:54:08.334 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph-authtool: stdout AQAPiBllCWOeOhAAzaL2xl2/tz+jET4OietnrA== 2023-10-01T14:54:08.904 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph-authtool: stdout AQAQiBll6rnNIBAARLc6Upn410rhWnUKWDZw2g== 2023-10-01T14:54:08.905 INFO:teuthology.orchestra.run.smithi031.stdout:Creating initial monmap... 2023-10-01T14:54:09.487 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-01T14:54:09.487 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2023-10-01T14:54:09.487 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:54:09.487 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-01T14:54:09.487 INFO:teuthology.orchestra.run.smithi031.stdout:monmaptool for a [v2:172.21.15.31:3300,v1:172.21.15.31:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-01T14:54:09.487 INFO:teuthology.orchestra.run.smithi031.stdout:setting min_mon_release = pacific 2023-10-01T14:54:09.487 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/monmaptool: set fsid to 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:54:09.487 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-01T14:54:09.487 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:54:09.488 INFO:teuthology.orchestra.run.smithi031.stdout:Creating mon... 2023-10-01T14:54:10.060 INFO:teuthology.orchestra.run.smithi031.stdout:create mon.a on 2023-10-01T14:54:10.297 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-10-01T14:54:10.445 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-466018a4-606a-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-466018a4-606a-11ee-8db5-212e2dc638e7.target. 2023-10-01T14:54:10.445 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-466018a4-606a-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-466018a4-606a-11ee-8db5-212e2dc638e7.target. 2023-10-01T14:54:10.630 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.a 2023-10-01T14:54:10.630 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stderr Failed to reset failed state of unit ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-01T14:54:10.731 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-466018a4-606a-11ee-8db5-212e2dc638e7.target.wants/ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-466018a4-606a-11ee-8db5-212e2dc638e7@.service. 2023-10-01T14:54:11.159 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-01T14:54:11.159 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout disabled 2023-10-01T14:54:11.170 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-01T14:54:11.170 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout inactive 2023-10-01T14:54:11.170 INFO:teuthology.orchestra.run.smithi031.stdout:firewalld.service is not enabled 2023-10-01T14:54:11.170 INFO:teuthology.orchestra.run.smithi031.stdout:Not possible to enable service . firewalld.service is not available 2023-10-01T14:54:11.170 INFO:teuthology.orchestra.run.smithi031.stdout:Waiting for mon to start... 2023-10-01T14:54:11.170 INFO:teuthology.orchestra.run.smithi031.stdout:Waiting for mon... 2023-10-01T14:54:11.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:11 smithi031 ceph-mon[106771]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-01T14:54:11.986 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout cluster: 2023-10-01T14:54:11.986 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout id: 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:54:11.986 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-10-01T14:54:11.987 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 2023-10-01T14:54:11.987 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout services: 2023-10-01T14:54:11.987 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.458621s) 2023-10-01T14:54:11.987 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-10-01T14:54:11.987 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-10-01T14:54:11.987 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 2023-10-01T14:54:11.987 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout data: 2023-10-01T14:54:11.987 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-10-01T14:54:11.987 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-10-01T14:54:11.987 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-10-01T14:54:11.987 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout pgs: 2023-10-01T14:54:11.987 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 2023-10-01T14:54:11.987 INFO:teuthology.orchestra.run.smithi031.stdout:mon is available 2023-10-01T14:54:11.988 INFO:teuthology.orchestra.run.smithi031.stdout:Assimilating anything we can from ceph.conf... 2023-10-01T14:54:12.202 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:12 smithi031 ceph-mon[106771]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-01T14:54:12.202 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:12 smithi031 ceph-mon[106771]: monmap e1: 1 mons at {a=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0]} removed_ranks: {} 2023-10-01T14:54:12.202 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:12 smithi031 ceph-mon[106771]: fsmap 2023-10-01T14:54:12.202 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:12 smithi031 ceph-mon[106771]: osdmap e1: 0 total, 0 up, 0 in 2023-10-01T14:54:12.202 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:12 smithi031 ceph-mon[106771]: mgrmap e1: no daemons active 2023-10-01T14:54:12.202 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:12 smithi031 ceph-mon[106771]: from='client.? 172.21.15.31:0/544202240' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-01T14:54:12.828 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 2023-10-01T14:54:12.828 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout [global] 2023-10-01T14:54:12.828 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout fsid = 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:54:12.828 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.31:3300,v1:172.21.15.31:6789] 2023-10-01T14:54:12.828 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-01T14:54:12.829 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-01T14:54:12.829 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-01T14:54:12.829 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-01T14:54:12.829 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 2023-10-01T14:54:12.829 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-01T14:54:12.829 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-01T14:54:12.829 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 2023-10-01T14:54:12.829 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout [osd] 2023-10-01T14:54:12.829 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-01T14:54:12.829 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-01T14:54:12.829 INFO:teuthology.orchestra.run.smithi031.stdout:Generating new minimal ceph.conf... 2023-10-01T14:54:13.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:13 smithi031 ceph-mon[106771]: from='client.? 172.21.15.31:0/279391791' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-01T14:54:13.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:13 smithi031 ceph-mon[106771]: from='client.? 172.21.15.31:0/279391791' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-01T14:54:13.594 INFO:teuthology.orchestra.run.smithi031.stdout:Restarting the monitor... 2023-10-01T14:54:13.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:13 smithi031 systemd[1]: Stopping Ceph mon.a for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T14:54:13.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:13 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mon-a[106767]: 2023-10-01T14:54:13.792+0000 7ffbdb82e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T14:54:13.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:13 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mon-a[106767]: 2023-10-01T14:54:13.792+0000 7ffbdb82e700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-01T14:54:14.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:14 smithi031 bash[107023]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mon-a 2023-10-01T14:54:15.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:15 smithi031 systemd[1]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-01T14:54:15.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:15 smithi031 systemd[1]: Stopped Ceph mon.a for 466018a4-606a-11ee-8db5-212e2dc638e7. 2023-10-01T14:54:15.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:15 smithi031 systemd[1]: Starting Ceph mon.a for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T14:54:16.218 INFO:teuthology.orchestra.run.smithi031.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2023-10-01T14:54:16.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: set uid:gid to 167:167 (ceph:ceph) 2023-10-01T14:54:16.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev), process ceph-mon, pid 2 2023-10-01T14:54:16.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: pidfile_write: ignore empty --pid-file 2023-10-01T14:54:16.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: load: jerasure load: lrc 2023-10-01T14:54:16.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: RocksDB version: 7.9.2 2023-10-01T14:54:16.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Git sha 0 2023-10-01T14:54:16.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Compile date 2023-09-25 19:01:54 2023-10-01T14:54:16.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: DB SUMMARY 2023-10-01T14:54:16.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: DB Session ID: 31YLYDTK1GFSNMC131Z3 2023-10-01T14:54:16.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: CURRENT file: CURRENT 2023-10-01T14:54:16.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: IDENTITY file: IDENTITY 2023-10-01T14:54:16.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2023-10-01T14:54:16.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-10-01T14:54:16.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 81738 ; 2023-10-01T14:54:16.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.error_if_exists: 0 2023-10-01T14:54:16.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.create_if_missing: 0 2023-10-01T14:54:16.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.paranoid_checks: 1 2023-10-01T14:54:16.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-01T14:54:16.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-01T14:54:16.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-01T14:54:16.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.env: 0x560f2a083da0 2023-10-01T14:54:16.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.fs: PosixFileSystem 2023-10-01T14:54:16.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.info_log: 0x560f2b983460 2023-10-01T14:54:16.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_file_opening_threads: 16 2023-10-01T14:54:16.498 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.statistics: (nil) 2023-10-01T14:54:16.498 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.use_fsync: 0 2023-10-01T14:54:16.498 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_log_file_size: 0 2023-10-01T14:54:16.498 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T14:54:16.498 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T14:54:16.498 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.keep_log_file_num: 1000 2023-10-01T14:54:16.498 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.recycle_log_file_num: 0 2023-10-01T14:54:16.498 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.allow_fallocate: 1 2023-10-01T14:54:16.498 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.allow_mmap_reads: 0 2023-10-01T14:54:16.498 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.allow_mmap_writes: 0 2023-10-01T14:54:16.498 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.use_direct_reads: 0 2023-10-01T14:54:16.498 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.create_missing_column_families: 0 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.db_log_dir: 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.wal_dir: 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.advise_random_on_open: 1 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.db_write_buffer_size: 0 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.write_buffer_manager: 0x560f2b93d4a0 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T14:54:16.499 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T14:54:16.500 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T14:54:16.500 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.rate_limiter: (nil) 2023-10-01T14:54:16.500 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T14:54:16.500 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.wal_recovery_mode: 2 2023-10-01T14:54:16.500 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.enable_thread_tracking: 0 2023-10-01T14:54:16.500 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.enable_pipelined_write: 0 2023-10-01T14:54:16.500 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.unordered_write: 0 2023-10-01T14:54:16.500 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T14:54:16.500 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T14:54:16.500 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T14:54:16.500 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T14:54:16.500 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.row_cache: None 2023-10-01T14:54:16.500 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.wal_filter: None 2023-10-01T14:54:16.501 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T14:54:16.501 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.allow_ingest_behind: 0 2023-10-01T14:54:16.501 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.two_write_queues: 0 2023-10-01T14:54:16.501 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.manual_wal_flush: 0 2023-10-01T14:54:16.501 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.wal_compression: 0 2023-10-01T14:54:16.501 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.atomic_flush: 0 2023-10-01T14:54:16.501 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T14:54:16.501 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T14:54:16.501 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T14:54:16.501 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.log_readahead_size: 0 2023-10-01T14:54:16.501 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-01T14:54:16.501 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.best_efforts_recovery: 0 2023-10-01T14:54:16.502 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-01T14:54:16.502 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-01T14:54:16.502 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.allow_data_in_errors: 0 2023-10-01T14:54:16.502 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.db_host_id: __hostname__ 2023-10-01T14:54:16.503 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-01T14:54:16.503 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_background_jobs: 2 2023-10-01T14:54:16.503 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_background_compactions: -1 2023-10-01T14:54:16.503 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_subcompactions: 1 2023-10-01T14:54:16.503 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T14:54:16.503 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T14:54:16.503 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T14:54:16.503 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_total_wal_size: 0 2023-10-01T14:54:16.504 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T14:54:16.504 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T14:54:16.504 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T14:54:16.504 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T14:54:16.504 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_open_files: -1 2023-10-01T14:54:16.504 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.bytes_per_sync: 0 2023-10-01T14:54:16.504 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T14:54:16.504 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T14:54:16.504 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compaction_readahead_size: 0 2023-10-01T14:54:16.504 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_background_flushes: -1 2023-10-01T14:54:16.504 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Compression algorithms supported: 2023-10-01T14:54:16.504 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T14:54:16.504 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: kZSTD supported: 0 2023-10-01T14:54:16.505 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: kXpressCompression supported: 0 2023-10-01T14:54:16.505 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: kLZ4HCCompression supported: 1 2023-10-01T14:54:16.505 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: kZlibCompression supported: 1 2023-10-01T14:54:16.505 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: kSnappyCompression supported: 1 2023-10-01T14:54:16.505 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: kLZ4Compression supported: 1 2023-10-01T14:54:16.505 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: kBZip2Compression supported: 0 2023-10-01T14:54:16.505 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T14:54:16.505 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-01T14:54:16.505 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2023-10-01T14:54:16.505 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-01T14:54:16.505 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T14:54:16.505 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.merge_operator: 2023-10-01T14:54:16.506 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compaction_filter: None 2023-10-01T14:54:16.506 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compaction_filter_factory: None 2023-10-01T14:54:16.506 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.sst_partitioner_factory: None 2023-10-01T14:54:16.506 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T14:54:16.506 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T14:54:16.506 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560f2b983560) 2023-10-01T14:54:16.506 INFO:journalctl@ceph.mon.a.smithi031.stdout: cache_index_and_filter_blocks: 1 2023-10-01T14:54:16.506 INFO:journalctl@ceph.mon.a.smithi031.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T14:54:16.506 INFO:journalctl@ceph.mon.a.smithi031.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T14:54:16.506 INFO:journalctl@ceph.mon.a.smithi031.stdout: pin_top_level_index_and_filter: 1 2023-10-01T14:54:16.506 INFO:journalctl@ceph.mon.a.smithi031.stdout: index_type: 0 2023-10-01T14:54:16.506 INFO:journalctl@ceph.mon.a.smithi031.stdout: data_block_index_type: 0 2023-10-01T14:54:16.507 INFO:journalctl@ceph.mon.a.smithi031.stdout: index_shortening: 1 2023-10-01T14:54:16.507 INFO:journalctl@ceph.mon.a.smithi031.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-01T14:54:16.507 INFO:journalctl@ceph.mon.a.smithi031.stdout: checksum: 4 2023-10-01T14:54:16.507 INFO:journalctl@ceph.mon.a.smithi031.stdout: no_block_cache: 0 2023-10-01T14:54:16.507 INFO:journalctl@ceph.mon.a.smithi031.stdout: block_cache: 0x560f2b9db1f0 2023-10-01T14:54:16.507 INFO:journalctl@ceph.mon.a.smithi031.stdout: block_cache_name: BinnedLRUCache 2023-10-01T14:54:16.507 INFO:journalctl@ceph.mon.a.smithi031.stdout: block_cache_options: 2023-10-01T14:54:16.507 INFO:journalctl@ceph.mon.a.smithi031.stdout: capacity : 536870912 2023-10-01T14:54:16.507 INFO:journalctl@ceph.mon.a.smithi031.stdout: num_shard_bits : 4 2023-10-01T14:54:16.507 INFO:journalctl@ceph.mon.a.smithi031.stdout: strict_capacity_limit : 0 2023-10-01T14:54:16.507 INFO:journalctl@ceph.mon.a.smithi031.stdout: high_pri_pool_ratio: 0.000 2023-10-01T14:54:16.507 INFO:journalctl@ceph.mon.a.smithi031.stdout: block_cache_compressed: (nil) 2023-10-01T14:54:16.507 INFO:journalctl@ceph.mon.a.smithi031.stdout: persistent_cache: (nil) 2023-10-01T14:54:16.508 INFO:journalctl@ceph.mon.a.smithi031.stdout: block_size: 4096 2023-10-01T14:54:16.508 INFO:journalctl@ceph.mon.a.smithi031.stdout: block_size_deviation: 10 2023-10-01T14:54:16.508 INFO:journalctl@ceph.mon.a.smithi031.stdout: block_restart_interval: 16 2023-10-01T14:54:16.508 INFO:journalctl@ceph.mon.a.smithi031.stdout: index_block_restart_interval: 1 2023-10-01T14:54:16.508 INFO:journalctl@ceph.mon.a.smithi031.stdout: metadata_block_size: 4096 2023-10-01T14:54:16.508 INFO:journalctl@ceph.mon.a.smithi031.stdout: partition_filters: 0 2023-10-01T14:54:16.508 INFO:journalctl@ceph.mon.a.smithi031.stdout: use_delta_encoding: 1 2023-10-01T14:54:16.508 INFO:journalctl@ceph.mon.a.smithi031.stdout: filter_policy: bloomfilter 2023-10-01T14:54:16.508 INFO:journalctl@ceph.mon.a.smithi031.stdout: whole_key_filtering: 1 2023-10-01T14:54:16.508 INFO:journalctl@ceph.mon.a.smithi031.stdout: verify_compression: 0 2023-10-01T14:54:16.508 INFO:journalctl@ceph.mon.a.smithi031.stdout: read_amp_bytes_per_bit: 0 2023-10-01T14:54:16.508 INFO:journalctl@ceph.mon.a.smithi031.stdout: format_version: 5 2023-10-01T14:54:16.508 INFO:journalctl@ceph.mon.a.smithi031.stdout: enable_index_compression: 1 2023-10-01T14:54:16.509 INFO:journalctl@ceph.mon.a.smithi031.stdout: block_align: 0 2023-10-01T14:54:16.509 INFO:journalctl@ceph.mon.a.smithi031.stdout: max_auto_readahead_size: 262144 2023-10-01T14:54:16.509 INFO:journalctl@ceph.mon.a.smithi031.stdout: prepopulate_block_cache: 0 2023-10-01T14:54:16.509 INFO:journalctl@ceph.mon.a.smithi031.stdout: initial_auto_readahead_size: 8192 2023-10-01T14:54:16.509 INFO:journalctl@ceph.mon.a.smithi031.stdout: num_file_reads_for_auto_readahead: 2 2023-10-01T14:54:16.509 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.write_buffer_size: 33554432 2023-10-01T14:54:16.509 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_write_buffer_number: 2 2023-10-01T14:54:16.509 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compression: NoCompression 2023-10-01T14:54:16.509 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.bottommost_compression: Disabled 2023-10-01T14:54:16.509 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.prefix_extractor: nullptr 2023-10-01T14:54:16.509 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T14:54:16.509 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.num_levels: 7 2023-10-01T14:54:16.510 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T14:54:16.510 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T14:54:16.510 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T14:54:16.510 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T14:54:16.510 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T14:54:16.510 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T14:54:16.510 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T14:54:16.510 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T14:54:16.510 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-01T14:54:16.510 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T14:54:16.510 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-01T14:54:16.510 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-01T14:54:16.511 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T14:54:16.511 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compression_opts.level: 32767 2023-10-01T14:54:16.511 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compression_opts.strategy: 0 2023-10-01T14:54:16.511 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T14:54:16.511 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T14:54:16.511 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-01T14:54:16.511 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-01T14:54:16.511 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compression_opts.enabled: false 2023-10-01T14:54:16.511 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-01T14:54:16.511 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T14:54:16.511 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T14:54:16.511 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T14:54:16.511 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.target_file_size_base: 67108864 2023-10-01T14:54:16.512 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T14:54:16.512 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T14:54:16.512 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T14:54:16.512 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T14:54:16.512 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T14:54:16.512 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T14:54:16.512 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T14:54:16.512 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T14:54:16.512 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T14:54:16.512 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T14:54:16.512 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T14:54:16.512 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T14:54:16.512 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T14:54:16.513 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-01T14:54:16.513 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.arena_block_size: 1048576 2023-10-01T14:54:16.513 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T14:54:16.513 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T14:54:16.513 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.disable_auto_compactions: 0 2023-10-01T14:54:16.513 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T14:54:16.513 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T14:54:16.513 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T14:54:16.513 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T14:54:16.513 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T14:54:16.513 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T14:54:16.513 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T14:54:16.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T14:54:16.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T14:54:16.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T14:54:16.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.table_properties_collectors: 2023-10-01T14:54:16.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.inplace_update_support: 0 2023-10-01T14:54:16.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T14:54:16.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T14:54:16.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T14:54:16.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T14:54:16.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.bloom_locality: 0 2023-10-01T14:54:16.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.max_successive_merges: 0 2023-10-01T14:54:16.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T14:54:16.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.paranoid_file_checks: 0 2023-10-01T14:54:16.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.force_consistency_checks: 1 2023-10-01T14:54:16.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.report_bg_io_stats: 0 2023-10-01T14:54:16.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.ttl: 2592000 2023-10-01T14:54:16.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T14:54:16.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-01T14:54:16.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-01T14:54:16.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.enable_blob_files: false 2023-10-01T14:54:16.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.min_blob_size: 0 2023-10-01T14:54:16.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.blob_file_size: 268435456 2023-10-01T14:54:16.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-01T14:54:16.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-01T14:54:16.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-01T14:54:16.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-01T14:54:16.516 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-01T14:54:16.516 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.blob_file_starting_level: 0 2023-10-01T14:54:16.516 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-01T14:54:16.516 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2023-10-01T14:54:16.516 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2023-10-01T14:54:16.516 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: cb46f6ff-d154-4e3b-9640-33164d450ebe 2023-10-01T14:54:16.516 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696172056198184, "job": 1, "event": "recovery_started", "wal_files": [9]} 2023-10-01T14:54:16.516 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2023-10-01T14:54:16.516 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696172056199177, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 78609, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 241, "table_properties": {"data_size": 76853, "index_size": 230, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 9697, "raw_average_key_size": 47, "raw_value_size": 71309, "raw_average_value_size": 349, "num_data_blocks": 10, "num_entries": 204, "num_filter_entries": 204, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696172056, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "cb46f6ff-d154-4e3b-9640-33164d450ebe", "db_session_id": "31YLYDTK1GFSNMC131Z3", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2023-10-01T14:54:16.516 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696172056199262, "job": 1, "event": "recovery_finished"} 2023-10-01T14:54:16.516 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2023-10-01T14:54:16.516 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-01T14:54:16.517 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x560f2ba78000 2023-10-01T14:54:16.517 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: DB pointer 0x560f2ba64000 2023-10-01T14:54:16.517 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-01T14:54:16.517 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-01T14:54:16.517 INFO:journalctl@ceph.mon.a.smithi031.stdout: ** DB Stats ** 2023-10-01T14:54:16.517 INFO:journalctl@ceph.mon.a.smithi031.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T14:54:16.517 INFO:journalctl@ceph.mon.a.smithi031.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T14:54:16.517 INFO:journalctl@ceph.mon.a.smithi031.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T14:54:16.517 INFO:journalctl@ceph.mon.a.smithi031.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T14:54:16.517 INFO:journalctl@ceph.mon.a.smithi031.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T14:54:16.517 INFO:journalctl@ceph.mon.a.smithi031.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T14:54:16.517 INFO:journalctl@ceph.mon.a.smithi031.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T14:54:16.517 INFO:journalctl@ceph.mon.a.smithi031.stdout: 2023-10-01T14:54:16.518 INFO:journalctl@ceph.mon.a.smithi031.stdout: ** Compaction Stats [default] ** 2023-10-01T14:54:16.518 INFO:journalctl@ceph.mon.a.smithi031.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T14:54:16.518 INFO:journalctl@ceph.mon.a.smithi031.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-01T14:54:16.518 INFO:journalctl@ceph.mon.a.smithi031.stdout: L0 2/0 78.60 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 120.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-01T14:54:16.518 INFO:journalctl@ceph.mon.a.smithi031.stdout: Sum 2/0 78.60 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 120.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-01T14:54:16.518 INFO:journalctl@ceph.mon.a.smithi031.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 120.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-01T14:54:16.518 INFO:journalctl@ceph.mon.a.smithi031.stdout: 2023-10-01T14:54:16.518 INFO:journalctl@ceph.mon.a.smithi031.stdout: ** Compaction Stats [default] ** 2023-10-01T14:54:16.518 INFO:journalctl@ceph.mon.a.smithi031.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T14:54:16.518 INFO:journalctl@ceph.mon.a.smithi031.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T14:54:16.518 INFO:journalctl@ceph.mon.a.smithi031.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 120.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-01T14:54:16.518 INFO:journalctl@ceph.mon.a.smithi031.stdout: 2023-10-01T14:54:16.519 INFO:journalctl@ceph.mon.a.smithi031.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-01T14:54:16.519 INFO:journalctl@ceph.mon.a.smithi031.stdout: 2023-10-01T14:54:16.519 INFO:journalctl@ceph.mon.a.smithi031.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T14:54:16.519 INFO:journalctl@ceph.mon.a.smithi031.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T14:54:16.519 INFO:journalctl@ceph.mon.a.smithi031.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T14:54:16.519 INFO:journalctl@ceph.mon.a.smithi031.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T14:54:16.519 INFO:journalctl@ceph.mon.a.smithi031.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T14:54:16.519 INFO:journalctl@ceph.mon.a.smithi031.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-01T14:54:16.519 INFO:journalctl@ceph.mon.a.smithi031.stdout: Cumulative compaction: 0.00 GB write, 21.87 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T14:54:16.519 INFO:journalctl@ceph.mon.a.smithi031.stdout: Interval compaction: 0.00 GB write, 21.87 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T14:54:16.519 INFO:journalctl@ceph.mon.a.smithi031.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T14:54:16.519 INFO:journalctl@ceph.mon.a.smithi031.stdout: Block cache BinnedLRUCache@0x560f2b9db1f0#2 capacity: 512.00 MB usage: 1.12 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-10-01T14:54:16.519 INFO:journalctl@ceph.mon.a.smithi031.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.70 KB,0.00013411%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2023-10-01T14:54:16.520 INFO:journalctl@ceph.mon.a.smithi031.stdout: 2023-10-01T14:54:16.520 INFO:journalctl@ceph.mon.a.smithi031.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-01T14:54:16.520 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: starting mon.a rank 0 at public addrs [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] at bind addrs [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:54:16.520 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: mon.a@-1(???) e1 preinit fsid 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:54:16.520 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: mon.a@-1(???).mds e1 new map 2023-10-01T14:54:16.520 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: mon.a@-1(???).mds e1 print_map 2023-10-01T14:54:16.520 INFO:journalctl@ceph.mon.a.smithi031.stdout: e1 2023-10-01T14:54:16.520 INFO:journalctl@ceph.mon.a.smithi031.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T14:54:16.520 INFO:journalctl@ceph.mon.a.smithi031.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments} 2023-10-01T14:54:16.520 INFO:journalctl@ceph.mon.a.smithi031.stdout: legacy client fscid: -1 2023-10-01T14:54:16.520 INFO:journalctl@ceph.mon.a.smithi031.stdout: 2023-10-01T14:54:16.520 INFO:journalctl@ceph.mon.a.smithi031.stdout: No filesystems configured 2023-10-01T14:54:16.520 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-01T14:54:16.521 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T14:54:16.521 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T14:54:16.521 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T14:54:16.521 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-01T14:54:16.521 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 bash[107133]: d2a8ce6ea8e5c59feb6cbb1d0bb90446fe4dda5e3e5b769a9c7d2ee9b571d9d1 2023-10-01T14:54:16.521 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 systemd[1]: Started Ceph mon.a for 466018a4-606a-11ee-8db5-212e2dc638e7. 2023-10-01T14:54:16.521 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-01T14:54:16.521 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: monmap e1: 1 mons at {a=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0]} removed_ranks: {} 2023-10-01T14:54:16.521 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: fsmap 2023-10-01T14:54:16.521 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: osdmap e1: 0 total, 0 up, 0 in 2023-10-01T14:54:16.521 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:16 smithi031 ceph-mon[107152]: mgrmap e1: no daemons active 2023-10-01T14:54:16.993 INFO:teuthology.orchestra.run.smithi031.stdout:Wrote config to /etc/ceph/ceph.conf 2023-10-01T14:54:17.019 INFO:teuthology.orchestra.run.smithi031.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-01T14:54:17.019 INFO:teuthology.orchestra.run.smithi031.stdout:Creating mgr... 2023-10-01T14:54:17.020 INFO:teuthology.orchestra.run.smithi031.stdout:Verifying port 0.0.0.0:9283 ... 2023-10-01T14:54:17.021 INFO:teuthology.orchestra.run.smithi031.stdout:Verifying port 0.0.0.0:8765 ... 2023-10-01T14:54:17.268 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mgr.x 2023-10-01T14:54:17.268 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stderr Failed to reset failed state of unit ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mgr.x.service: Unit ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mgr.x.service not loaded. 2023-10-01T14:54:17.376 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-466018a4-606a-11ee-8db5-212e2dc638e7.target.wants/ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mgr.x.service → /etc/systemd/system/ceph-466018a4-606a-11ee-8db5-212e2dc638e7@.service. 2023-10-01T14:54:17.495 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:17 smithi031 systemd[1]: Starting Ceph mgr.x for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T14:54:17.756 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-01T14:54:17.756 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout disabled 2023-10-01T14:54:17.766 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-01T14:54:17.766 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout inactive 2023-10-01T14:54:17.766 INFO:teuthology.orchestra.run.smithi031.stdout:firewalld.service is not enabled 2023-10-01T14:54:17.766 INFO:teuthology.orchestra.run.smithi031.stdout:Not possible to enable service . firewalld.service is not available 2023-10-01T14:54:17.776 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-01T14:54:17.776 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout disabled 2023-10-01T14:54:17.786 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-01T14:54:17.786 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout inactive 2023-10-01T14:54:17.786 INFO:teuthology.orchestra.run.smithi031.stdout:firewalld.service is not enabled 2023-10-01T14:54:17.786 INFO:teuthology.orchestra.run.smithi031.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2023-10-01T14:54:17.786 INFO:teuthology.orchestra.run.smithi031.stdout:Waiting for mgr to start... 2023-10-01T14:54:17.786 INFO:teuthology.orchestra.run.smithi031.stdout:Waiting for mgr... 2023-10-01T14:54:17.874 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:17 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/1071239230' entity='client.admin' 2023-10-01T14:54:17.874 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:17 smithi031 bash[107363]: 44704039a3c5aa93d6b91e9ceab123a50c92ee6c98690511afa56f824c6beb23 2023-10-01T14:54:17.874 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:17 smithi031 systemd[1]: Started Ceph mgr.x for 466018a4-606a-11ee-8db5-212e2dc638e7. 2023-10-01T14:54:18.242 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:17 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:17.871+0000 7f1bafa41200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T14:54:18.243 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:18 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:18.073+0000 7f1bafa41200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T14:54:18.494 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:18 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:18.462+0000 7f1bafa41200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T14:54:18.603 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 2023-10-01T14:54:18.603 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout { 2023-10-01T14:54:18.603 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "fsid": "466018a4-606a-11ee-8db5-212e2dc638e7", 2023-10-01T14:54:18.603 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "health": { 2023-10-01T14:54:18.603 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 0 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout ], 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "a" 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout ], 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "quorum_age": 1, 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:18.604 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-10-01T14:54:18.605 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-01T14:54:18.605 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:18.605 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-01T14:54:18.605 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:18.605 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-01T14:54:18.605 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-01T14:54:18.605 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-01T14:54:18.605 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-01T14:54:18.605 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-01T14:54:18.605 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-01T14:54:18.605 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:18.605 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-01T14:54:18.606 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-01T14:54:18.606 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-01T14:54:18.606 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-01T14:54:18.606 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-01T14:54:18.606 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-01T14:54:18.606 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-01T14:54:18.606 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-01T14:54:18.606 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-01T14:54:18.606 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:18.606 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-01T14:54:18.606 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:18.606 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-01T14:54:18.607 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-01T14:54:18.607 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:18.607 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-01T14:54:18.607 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-01T14:54:18.607 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-01T14:54:18.607 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-01T14:54:18.607 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-01T14:54:18.607 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-01T14:54:18.607 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "restful" 2023-10-01T14:54:18.607 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout ], 2023-10-01T14:54:18.607 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-01T14:54:18.607 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:18.607 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-01T14:54:18.608 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:18.608 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "modified": "2023-10-01T14:54:11.181584+0000", 2023-10-01T14:54:18.608 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-01T14:54:18.608 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:18.608 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-01T14:54:18.608 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout } 2023-10-01T14:54:18.608 INFO:teuthology.orchestra.run.smithi031.stdout:mgr not available, waiting (1/15)... 2023-10-01T14:54:18.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:18 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/518478068' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-01T14:54:19.295 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:19 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:19.038+0000 7f1bafa41200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T14:54:20.245 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:19 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:19.294+0000 7f1bafa41200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T14:54:20.245 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:19 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:19.773+0000 7f1bafa41200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T14:54:20.995 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:20 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:20.591+0000 7f1bafa41200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T14:54:20.995 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:20 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:20.699+0000 7f1bafa41200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T14:54:21.369 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:21 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/146554661' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-01T14:54:21.401 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 2023-10-01T14:54:21.401 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout { 2023-10-01T14:54:21.401 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "fsid": "466018a4-606a-11ee-8db5-212e2dc638e7", 2023-10-01T14:54:21.402 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "health": { 2023-10-01T14:54:21.402 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-01T14:54:21.402 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-01T14:54:21.402 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-01T14:54:21.402 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:21.402 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-01T14:54:21.403 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-01T14:54:21.403 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 0 2023-10-01T14:54:21.403 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout ], 2023-10-01T14:54:21.403 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-01T14:54:21.403 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "a" 2023-10-01T14:54:21.403 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout ], 2023-10-01T14:54:21.403 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "quorum_age": 4, 2023-10-01T14:54:21.404 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-01T14:54:21.404 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:21.404 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-10-01T14:54:21.404 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-01T14:54:21.404 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:21.404 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-01T14:54:21.404 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:21.405 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-01T14:54:21.405 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-01T14:54:21.405 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-01T14:54:21.405 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-01T14:54:21.408 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-01T14:54:21.408 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-01T14:54:21.408 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:21.408 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-01T14:54:21.408 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-01T14:54:21.409 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-01T14:54:21.409 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-01T14:54:21.409 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-01T14:54:21.409 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-01T14:54:21.409 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-01T14:54:21.409 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-01T14:54:21.409 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-01T14:54:21.409 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:21.410 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-01T14:54:21.410 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:21.410 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-01T14:54:21.410 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-01T14:54:21.410 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:21.410 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-01T14:54:21.410 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-01T14:54:21.411 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-01T14:54:21.411 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-01T14:54:21.411 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-01T14:54:21.411 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-01T14:54:21.411 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "restful" 2023-10-01T14:54:21.411 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout ], 2023-10-01T14:54:21.411 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-01T14:54:21.411 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:21.412 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-01T14:54:21.412 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:21.412 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "modified": "2023-10-01T14:54:11.181584+0000", 2023-10-01T14:54:21.412 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-01T14:54:21.412 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:21.412 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-01T14:54:21.412 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout } 2023-10-01T14:54:21.413 INFO:teuthology.orchestra.run.smithi031.stdout:mgr not available, waiting (2/15)... 2023-10-01T14:54:21.684 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:21 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:21.408+0000 7f1bafa41200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T14:54:21.684 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:21 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:21.502+0000 7f1bafa41200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T14:54:21.684 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:21 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:21.602+0000 7f1bafa41200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T14:54:21.966 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:21 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:21.861+0000 7f1bafa41200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T14:54:22.244 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:21 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:21.965+0000 7f1bafa41200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T14:54:22.245 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:22 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:22.069+0000 7f1bafa41200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T14:54:22.626 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:22 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:22.310+0000 7f1bafa41200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T14:54:22.626 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:22 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:22.418+0000 7f1bafa41200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T14:54:22.994 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:22 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:22.625+0000 7f1bafa41200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T14:54:22.994 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:22 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:22.721+0000 7f1bafa41200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T14:54:23.813 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:23 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:23.594+0000 7f1bafa41200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T14:54:24.110 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:23 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/2014661497' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-01T14:54:24.110 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:23 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:23.819+0000 7f1bafa41200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T14:54:24.111 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:24 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:24.022+0000 7f1bafa41200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T14:54:24.176 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 2023-10-01T14:54:24.176 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout { 2023-10-01T14:54:24.177 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "fsid": "466018a4-606a-11ee-8db5-212e2dc638e7", 2023-10-01T14:54:24.177 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "health": { 2023-10-01T14:54:24.177 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-01T14:54:24.177 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-01T14:54:24.177 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-01T14:54:24.177 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:24.177 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-01T14:54:24.177 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-01T14:54:24.177 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 0 2023-10-01T14:54:24.177 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout ], 2023-10-01T14:54:24.177 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-01T14:54:24.177 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "a" 2023-10-01T14:54:24.178 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout ], 2023-10-01T14:54:24.178 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "quorum_age": 7, 2023-10-01T14:54:24.178 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-01T14:54:24.178 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:24.178 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-10-01T14:54:24.178 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-01T14:54:24.178 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:24.178 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-01T14:54:24.178 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:24.178 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-01T14:54:24.178 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-01T14:54:24.178 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-01T14:54:24.179 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-01T14:54:24.182 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-01T14:54:24.182 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-01T14:54:24.182 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:24.182 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-01T14:54:24.182 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-01T14:54:24.182 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-01T14:54:24.182 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-01T14:54:24.182 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-01T14:54:24.183 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-01T14:54:24.183 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-01T14:54:24.183 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-01T14:54:24.183 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-01T14:54:24.183 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:24.183 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-01T14:54:24.183 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:24.183 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-01T14:54:24.183 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-01T14:54:24.183 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:24.183 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-01T14:54:24.184 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-01T14:54:24.184 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-01T14:54:24.184 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-01T14:54:24.184 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-01T14:54:24.184 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-01T14:54:24.184 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "restful" 2023-10-01T14:54:24.184 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout ], 2023-10-01T14:54:24.184 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-01T14:54:24.184 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:24.184 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-01T14:54:24.184 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:24.184 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "modified": "2023-10-01T14:54:11.181584+0000", 2023-10-01T14:54:24.185 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-01T14:54:24.185 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:24.185 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-01T14:54:24.185 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout } 2023-10-01T14:54:24.185 INFO:teuthology.orchestra.run.smithi031.stdout:mgr not available, waiting (3/15)... 2023-10-01T14:54:25.180 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:24 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:24.772+0000 7f1bafa41200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T14:54:25.180 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:24 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:24.974+0000 7f1bafa41200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T14:54:25.494 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:25.178+0000 7f1bafa41200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T14:54:25.495 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:25.308+0000 7f1bafa41200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T14:54:25.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-mon[107152]: Activating manager daemon x 2023-10-01T14:54:25.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-mon[107152]: mgrmap e2: x(active, starting, since 0.00349343s) 2023-10-01T14:54:25.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-mon[107152]: from='mgr.14100 172.21.15.31:0/4215742326' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T14:54:25.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-mon[107152]: from='mgr.14100 172.21.15.31:0/4215742326' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T14:54:25.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-mon[107152]: from='mgr.14100 172.21.15.31:0/4215742326' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T14:54:25.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-mon[107152]: from='mgr.14100 172.21.15.31:0/4215742326' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:54:25.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-mon[107152]: from='mgr.14100 172.21.15.31:0/4215742326' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T14:54:25.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-mon[107152]: Manager daemon x is now available 2023-10-01T14:54:25.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-mon[107152]: from='mgr.14100 172.21.15.31:0/4215742326' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T14:54:25.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-mon[107152]: from='mgr.14100 172.21.15.31:0/4215742326' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T14:54:25.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-mon[107152]: from='mgr.14100 172.21.15.31:0/4215742326' entity='mgr.x' 2023-10-01T14:54:25.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-mon[107152]: from='mgr.14100 172.21.15.31:0/4215742326' entity='mgr.x' 2023-10-01T14:54:25.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:25 smithi031 ceph-mon[107152]: from='mgr.14100 172.21.15.31:0/4215742326' entity='mgr.x' 2023-10-01T14:54:27.577 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:27 smithi031 ceph-mon[107152]: mgrmap e3: x(active, since 1.00979s) 2023-10-01T14:54:27.577 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:27 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/1573762327' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-01T14:54:27.609 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 2023-10-01T14:54:27.609 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout { 2023-10-01T14:54:27.609 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "fsid": "466018a4-606a-11ee-8db5-212e2dc638e7", 2023-10-01T14:54:27.609 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "health": { 2023-10-01T14:54:27.609 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-01T14:54:27.609 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-01T14:54:27.609 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-01T14:54:27.609 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:27.609 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-01T14:54:27.609 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-01T14:54:27.609 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 0 2023-10-01T14:54:27.610 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout ], 2023-10-01T14:54:27.610 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-01T14:54:27.610 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "a" 2023-10-01T14:54:27.610 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout ], 2023-10-01T14:54:27.610 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "quorum_age": 10, 2023-10-01T14:54:27.610 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-01T14:54:27.610 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:27.610 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-10-01T14:54:27.610 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-01T14:54:27.610 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:27.610 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-01T14:54:27.610 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:27.610 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-01T14:54:27.611 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-01T14:54:27.611 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-01T14:54:27.611 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-01T14:54:27.611 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-01T14:54:27.611 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-01T14:54:27.611 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:27.611 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-01T14:54:27.611 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-01T14:54:27.611 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-01T14:54:27.611 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-01T14:54:27.611 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-01T14:54:27.611 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-01T14:54:27.611 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-01T14:54:27.612 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-01T14:54:27.612 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-01T14:54:27.612 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:27.612 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-01T14:54:27.612 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:27.612 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-01T14:54:27.612 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-01T14:54:27.612 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:27.612 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-01T14:54:27.612 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-01T14:54:27.612 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-01T14:54:27.612 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "restful" 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout ], 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "modified": "2023-10-01T14:54:11.181584+0000", 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout }, 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout } 2023-10-01T14:54:27.613 INFO:teuthology.orchestra.run.smithi031.stdout:mgr is available 2023-10-01T14:54:28.435 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:28 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/563213441' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-01T14:54:28.483 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 2023-10-01T14:54:28.483 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout [global] 2023-10-01T14:54:28.483 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout fsid = 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:54:28.483 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-01T14:54:28.483 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-01T14:54:28.484 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-01T14:54:28.484 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-01T14:54:28.484 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 2023-10-01T14:54:28.484 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-01T14:54:28.484 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-01T14:54:28.484 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 2023-10-01T14:54:28.484 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout [osd] 2023-10-01T14:54:28.484 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-01T14:54:28.484 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-01T14:54:28.484 INFO:teuthology.orchestra.run.smithi031.stdout:Enabling cephadm module... 2023-10-01T14:54:29.664 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:29 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: ignoring --setuser ceph since I am not root 2023-10-01T14:54:29.664 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:29 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: ignoring --setgroup ceph since I am not root 2023-10-01T14:54:29.664 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:29 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:29.527+0000 7f7c3fb37200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T14:54:29.666 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:29 smithi031 ceph-mon[107152]: mgrmap e4: x(active, since 3s) 2023-10-01T14:54:29.666 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:29 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/2929264197' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-01T14:54:29.994 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:29 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:29.730+0000 7f7c3fb37200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T14:54:30.494 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:30 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:30.109+0000 7f7c3fb37200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T14:54:30.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:30 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/2929264197' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-01T14:54:30.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:30 smithi031 ceph-mon[107152]: mgrmap e5: x(active, since 4s) 2023-10-01T14:54:30.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:30 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/4186464801' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-01T14:54:30.671 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout { 2023-10-01T14:54:30.671 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-10-01T14:54:30.672 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-01T14:54:30.672 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "active_name": "x", 2023-10-01T14:54:30.672 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-01T14:54:30.672 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout } 2023-10-01T14:54:30.672 INFO:teuthology.orchestra.run.smithi031.stdout:Waiting for the mgr to restart... 2023-10-01T14:54:30.672 INFO:teuthology.orchestra.run.smithi031.stdout:Waiting for mgr epoch 5... 2023-10-01T14:54:30.809 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:30 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:30.680+0000 7f7c3fb37200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T14:54:31.244 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:30 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:30.941+0000 7f7c3fb37200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T14:54:31.745 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:31 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:31.418+0000 7f7c3fb37200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T14:54:32.494 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:32 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:32.229+0000 7f7c3fb37200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T14:54:32.494 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:32 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:32.329+0000 7f7c3fb37200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T14:54:33.494 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:33 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:33.027+0000 7f7c3fb37200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T14:54:33.495 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:33 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:33.123+0000 7f7c3fb37200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T14:54:33.495 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:33 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:33.226+0000 7f7c3fb37200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T14:54:33.953 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:33 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:33.496+0000 7f7c3fb37200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T14:54:33.953 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:33 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:33.600+0000 7f7c3fb37200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T14:54:33.953 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:33 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:33.707+0000 7f7c3fb37200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T14:54:33.953 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:33 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:33.950+0000 7f7c3fb37200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T14:54:34.244 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:34 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:34.060+0000 7f7c3fb37200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T14:54:34.744 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:34 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:34.270+0000 7f7c3fb37200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T14:54:34.745 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:34 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:34.368+0000 7f7c3fb37200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T14:54:35.495 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:35 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:35.236+0000 7f7c3fb37200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T14:54:35.495 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:35 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:35.457+0000 7f7c3fb37200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T14:54:35.994 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:35 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:35.658+0000 7f7c3fb37200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T14:54:36.744 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:36 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:36.413+0000 7f7c3fb37200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T14:54:36.744 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:36 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:36.619+0000 7f7c3fb37200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T14:54:37.212 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:36 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:36.828+0000 7f7c3fb37200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T14:54:37.213 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:36 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:36.959+0000 7f7c3fb37200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T14:54:37.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: Active manager daemon x restarted 2023-10-01T14:54:37.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: Activating manager daemon x 2023-10-01T14:54:37.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: osdmap e2: 0 total, 0 up, 0 in 2023-10-01T14:54:37.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: mgrmap e6: x(active, starting, since 0.00371782s) 2023-10-01T14:54:37.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:54:37.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T14:54:37.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T14:54:37.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T14:54:37.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T14:54:37.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: Manager daemon x is now available 2023-10-01T14:54:37.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:37.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:37.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:54:37.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:54:37.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:54:37.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T14:54:37.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:37 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T14:54:38.299 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:38 smithi031 ceph-mon[107152]: Found migration_current of "None". Setting to last migration. 2023-10-01T14:54:38.300 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:38 smithi031 ceph-mon[107152]: mgrmap e7: x(active, since 1.0056s) 2023-10-01T14:54:38.364 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout { 2023-10-01T14:54:38.364 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-10-01T14:54:38.364 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-01T14:54:38.365 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout } 2023-10-01T14:54:38.365 INFO:teuthology.orchestra.run.smithi031.stdout:mgr epoch 5 is available 2023-10-01T14:54:38.365 INFO:teuthology.orchestra.run.smithi031.stdout:Setting orchestrator backend to cephadm... 2023-10-01T14:54:39.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:39 smithi031 ceph-mon[107152]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-01T14:54:39.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:39 smithi031 ceph-mon[107152]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-01T14:54:39.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:39 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:39.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:39 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:39.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:39 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:39.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:39 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:54:40.164 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout value unchanged 2023-10-01T14:54:40.164 INFO:teuthology.orchestra.run.smithi031.stdout:Generating ssh key... 2023-10-01T14:54:40.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:40 smithi031 ceph-mon[107152]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:54:40.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:40 smithi031 ceph-mon[107152]: mgrmap e8: x(active, since 2s) 2023-10-01T14:54:41.358 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-mon[107152]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:54:41.358 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-mon[107152]: [01/Oct/2023:14:54:40] ENGINE Bus STARTING 2023-10-01T14:54:41.358 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-mon[107152]: [01/Oct/2023:14:54:40] ENGINE Serving on https://172.21.15.31:7150 2023-10-01T14:54:41.358 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-mon[107152]: [01/Oct/2023:14:54:40] ENGINE Serving on http://172.21.15.31:8765 2023-10-01T14:54:41.359 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-mon[107152]: [01/Oct/2023:14:54:40] ENGINE Bus STARTED 2023-10-01T14:54:41.359 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:54:41.359 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:41.359 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:41.359 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: Generating public/private rsa key pair. 2023-10-01T14:54:41.359 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: Your identification has been saved in /tmp/tmpxfk7xaqy/key. 2023-10-01T14:54:41.359 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: Your public key has been saved in /tmp/tmpxfk7xaqy/key.pub. 2023-10-01T14:54:41.359 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: The key fingerprint is: 2023-10-01T14:54:41.359 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: SHA256:tbLCDNeCeQ2FmtzUdLGki43c1xOY/Bb60+OB5U223p8 ceph-466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:54:41.360 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: The key's randomart image is: 2023-10-01T14:54:41.360 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: +---[RSA 3072]----+ 2023-10-01T14:54:41.360 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: | +o +. | 2023-10-01T14:54:41.360 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: | o..= + | 2023-10-01T14:54:41.360 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: | . =. . * o | 2023-10-01T14:54:41.360 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: | ++.O o = o | 2023-10-01T14:54:41.360 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: | + B S + = . o| 2023-10-01T14:54:41.360 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: | * . + o * +.| 2023-10-01T14:54:41.360 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: | + . + =..| 2023-10-01T14:54:41.360 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: | . o.oo| 2023-10-01T14:54:41.360 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: | .E+| 2023-10-01T14:54:41.360 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:41 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: +----[SHA256]-----+ 2023-10-01T14:54:42.339 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDxbDyQLuaqT2kOd0OlfKrtcoakTzSZT4BaYaY1sFiYLrr8S91iqfF38UgBvBJknBQHJAyX2T6SggI4hGNTLPpMCMtGGmn8hY5y6iMml6I61x8zPgZVs5EF4DZguSXq4VM7SXLHy/fs6dDmJ3Ir0hS9BuyjEI1PzwEcdf9uz8Q4AH7JNjtnDsxTAxAl1ZZSp8hgUvCei9eiACxBGzQu1Nl0D9wefSxlPRV3qCXK/5p9khGZ4nITDxDn63piccD06zQq2P+xIbmPQzIt3zGltJHEXUDqSkvMv+VjDm9crb51QPsSdWPMr8psGv0mOcI0iMj8KOhMWhsw1YIv/IEIyxEiXS6fUl8CLptE1jyCyWJUcZtyXq8NsleCDTBFmcTk9mNbKnMocwBf5d9cUiaRFRc108zmz6HOtpTO70t/JpluB5ZLsue2cu/aeQ2TQ1LDgVeRX5VBMtnDzTikCpCphEpZnBldSRZu67wQIa/kh8wFsSWRzBTOr4Wk9fpNLZGF1jU= ceph-466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:54:42.339 INFO:teuthology.orchestra.run.smithi031.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-01T14:54:42.339 INFO:teuthology.orchestra.run.smithi031.stdout:Adding key to root@localhost authorized_keys... 2023-10-01T14:54:42.340 INFO:teuthology.orchestra.run.smithi031.stdout:Adding host smithi031... 2023-10-01T14:54:42.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:42 smithi031 ceph-mon[107152]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:54:42.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:42 smithi031 ceph-mon[107152]: Generating ssh key... 2023-10-01T14:54:43.447 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:43 smithi031 ceph-mon[107152]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:54:44.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:44 smithi031 ceph-mon[107152]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi031", "addr": "172.21.15.31", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:54:45.612 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout Added host 'smithi031' with addr '172.21.15.31' 2023-10-01T14:54:45.612 INFO:teuthology.orchestra.run.smithi031.stdout:Deploying unmanaged mon service... 2023-10-01T14:54:45.616 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:45 smithi031 ceph-mon[107152]: Deploying cephadm binary to smithi031 2023-10-01T14:54:46.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:46 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:46.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:46 smithi031 ceph-mon[107152]: Added host smithi031 2023-10-01T14:54:46.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:46 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:54:46.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:46 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:46.696 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-10-01T14:54:46.696 INFO:teuthology.orchestra.run.smithi031.stdout:Deploying unmanaged mgr service... 2023-10-01T14:54:47.562 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:47 smithi031 ceph-mon[107152]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:54:47.562 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:47 smithi031 ceph-mon[107152]: Saving service mon spec with placement count:5 2023-10-01T14:54:47.562 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:47 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:48.071 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-10-01T14:54:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:48 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:48.746 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:48 smithi031 ceph-mon[107152]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:54:48.746 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:48 smithi031 ceph-mon[107152]: Saving service mgr spec with placement count:2 2023-10-01T14:54:48.746 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:48 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:49.944 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:49 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/139748351' entity='client.admin' 2023-10-01T14:54:49.944 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:49 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:50.128 INFO:teuthology.orchestra.run.smithi031.stdout:Enabling the dashboard module... 2023-10-01T14:54:50.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:50 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/618405221' entity='client.admin' 2023-10-01T14:54:50.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:50 smithi031 ceph-mon[107152]: from='mgr.14120 172.21.15.31:0/1442672623' entity='mgr.x' 2023-10-01T14:54:50.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:50 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/1599139712' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-01T14:54:51.393 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:51 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: ignoring --setuser ceph since I am not root 2023-10-01T14:54:51.393 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:51 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: ignoring --setgroup ceph since I am not root 2023-10-01T14:54:51.393 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:51 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:51.192+0000 7fe3bc752200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T14:54:51.744 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:51 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:51.390+0000 7fe3bc752200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T14:54:52.062 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:51 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:51.782+0000 7fe3bc752200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T14:54:52.063 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:51 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/1599139712' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-01T14:54:52.063 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:51 smithi031 ceph-mon[107152]: mgrmap e9: x(active, since 14s) 2023-10-01T14:54:52.395 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout { 2023-10-01T14:54:52.395 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-10-01T14:54:52.395 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-01T14:54:52.395 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "active_name": "x", 2023-10-01T14:54:52.395 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-01T14:54:52.396 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout } 2023-10-01T14:54:52.396 INFO:teuthology.orchestra.run.smithi031.stdout:Waiting for the mgr to restart... 2023-10-01T14:54:52.396 INFO:teuthology.orchestra.run.smithi031.stdout:Waiting for mgr epoch 9... 2023-10-01T14:54:52.626 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:52 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:52.356+0000 7fe3bc752200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T14:54:52.992 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:52 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:52.621+0000 7fe3bc752200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T14:54:53.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:52 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/2093111041' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-01T14:54:53.245 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:53 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:53.103+0000 7fe3bc752200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T14:54:54.245 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:53 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:53.908+0000 7fe3bc752200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T14:54:54.245 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:54 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:54.006+0000 7fe3bc752200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T14:54:54.994 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:54 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:54.696+0000 7fe3bc752200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T14:54:54.995 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:54 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:54.792+0000 7fe3bc752200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T14:54:54.995 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:54 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:54.892+0000 7fe3bc752200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T14:54:55.494 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:55 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:55.152+0000 7fe3bc752200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T14:54:55.494 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:55 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:55.256+0000 7fe3bc752200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T14:54:55.495 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:55 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:55.362+0000 7fe3bc752200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T14:54:55.923 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:55 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:55.603+0000 7fe3bc752200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T14:54:55.923 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:55 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:55.712+0000 7fe3bc752200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T14:54:56.244 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:55 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:55.921+0000 7fe3bc752200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T14:54:56.245 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:56 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:56.018+0000 7fe3bc752200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T14:54:57.244 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:56 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:56.876+0000 7fe3bc752200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T14:54:57.245 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:57 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:57.095+0000 7fe3bc752200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T14:54:57.744 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:57 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:57.297+0000 7fe3bc752200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T14:54:58.460 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:58.048+0000 7fe3bc752200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T14:54:58.460 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:58.252+0000 7fe3bc752200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T14:54:58.726 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:58.458+0000 7fe3bc752200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T14:54:58.726 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:54:58.587+0000 7fe3bc752200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T14:54:58.726 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-mon[107152]: Active manager daemon x restarted 2023-10-01T14:54:58.727 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-mon[107152]: Activating manager daemon x 2023-10-01T14:54:58.727 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-mon[107152]: osdmap e3: 0 total, 0 up, 0 in 2023-10-01T14:54:58.727 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-mon[107152]: mgrmap e10: x(active, starting, since 0.00368969s) 2023-10-01T14:54:58.727 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:54:58.727 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T14:54:58.727 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T14:54:58.727 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T14:54:58.727 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T14:54:58.727 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-mon[107152]: Manager daemon x is now available 2023-10-01T14:54:58.727 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:54:58.727 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:58 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:54:59.973 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T14:54:59.973 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T14:54:59.973 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:54:59 smithi031 ceph-mon[107152]: mgrmap e11: x(active, since 1.00732s) 2023-10-01T14:55:00.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:00 smithi031 ceph-mon[107152]: [01/Oct/2023:14:54:59] ENGINE Bus STARTING 2023-10-01T14:55:00.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:00 smithi031 ceph-mon[107152]: [01/Oct/2023:14:54:59] ENGINE Serving on https://172.21.15.31:7150 2023-10-01T14:55:00.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:00 smithi031 ceph-mon[107152]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-01T14:55:00.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:00 smithi031 ceph-mon[107152]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-01T14:55:00.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:00 smithi031 ceph-mon[107152]: [01/Oct/2023:14:54:59] ENGINE Serving on http://172.21.15.31:8765 2023-10-01T14:55:00.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:00 smithi031 ceph-mon[107152]: [01/Oct/2023:14:54:59] ENGINE Bus STARTED 2023-10-01T14:55:01.360 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout { 2023-10-01T14:55:01.360 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-10-01T14:55:01.361 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-01T14:55:01.361 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout } 2023-10-01T14:55:01.361 INFO:teuthology.orchestra.run.smithi031.stdout:mgr epoch 9 is available 2023-10-01T14:55:01.361 INFO:teuthology.orchestra.run.smithi031.stdout:Generating a dashboard self-signed certificate... 2023-10-01T14:55:01.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:01 smithi031 ceph-mon[107152]: mgrmap e12: x(active, since 2s) 2023-10-01T14:55:02.409 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-10-01T14:55:02.410 INFO:teuthology.orchestra.run.smithi031.stdout:Creating initial admin user... 2023-10-01T14:55:03.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:03.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:02 smithi031 ceph-mon[107152]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:03.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:03.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:03.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:03.710 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$Zf4gYEkKt.Idul8pusynju./JQlN404gekaEQBfXMfbcVly8zf5/6", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696172103, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-01T14:55:03.710 INFO:teuthology.orchestra.run.smithi031.stdout:Fetching dashboard port number... 2023-10-01T14:55:03.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:03 smithi031 ceph-mon[107152]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:03.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:03 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:04.577 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stdout 8443 2023-10-01T14:55:04.588 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-01T14:55:04.588 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout disabled 2023-10-01T14:55:04.598 INFO:teuthology.orchestra.run.smithi031.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-01T14:55:04.599 INFO:teuthology.orchestra.run.smithi031.stdout:systemctl: stdout inactive 2023-10-01T14:55:04.599 INFO:teuthology.orchestra.run.smithi031.stdout:firewalld.service is not enabled 2023-10-01T14:55:04.599 INFO:teuthology.orchestra.run.smithi031.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-01T14:55:04.600 INFO:teuthology.orchestra.run.smithi031.stdout:Ceph Dashboard is now available at: 2023-10-01T14:55:04.600 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:55:04.601 INFO:teuthology.orchestra.run.smithi031.stdout: URL: https://smithi031.front.sepia.ceph.com:8443/ 2023-10-01T14:55:04.601 INFO:teuthology.orchestra.run.smithi031.stdout: User: admin 2023-10-01T14:55:04.601 INFO:teuthology.orchestra.run.smithi031.stdout: Password: eq7a31stx8 2023-10-01T14:55:04.601 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:55:04.601 INFO:teuthology.orchestra.run.smithi031.stdout:Saving cluster configuration to /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config directory 2023-10-01T14:55:04.602 INFO:teuthology.orchestra.run.smithi031.stdout:Enabling autotune for osd_memory_target 2023-10-01T14:55:05.184 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:04 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/3899194437' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-01T14:55:06.188 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:05 smithi031 ceph-mon[107152]: mgrmap e13: x(active, since 6s) 2023-10-01T14:55:06.518 INFO:teuthology.orchestra.run.smithi031.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-10-01T14:55:06.518 INFO:teuthology.orchestra.run.smithi031.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-01T14:55:06.518 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:55:06.519 INFO:teuthology.orchestra.run.smithi031.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-01T14:55:06.519 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:55:06.519 INFO:teuthology.orchestra.run.smithi031.stdout:Or, if you are only running a single cluster on this host: 2023-10-01T14:55:06.519 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:55:06.519 INFO:teuthology.orchestra.run.smithi031.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-01T14:55:06.519 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:55:06.519 INFO:teuthology.orchestra.run.smithi031.stdout:Please consider enabling telemetry to help improve Ceph: 2023-10-01T14:55:06.519 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:55:06.519 INFO:teuthology.orchestra.run.smithi031.stdout: ceph telemetry on 2023-10-01T14:55:06.519 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:55:06.519 INFO:teuthology.orchestra.run.smithi031.stdout:For more information see: 2023-10-01T14:55:06.519 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:55:06.520 INFO:teuthology.orchestra.run.smithi031.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2023-10-01T14:55:06.520 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:55:06.520 INFO:teuthology.orchestra.run.smithi031.stdout:Bootstrap complete. 2023-10-01T14:55:06.561 INFO:tasks.cephadm:Fetching config... 2023-10-01T14:55:06.561 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2023-10-01T14:55:06.562 DEBUG:teuthology.orchestra.run.smithi031:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-01T14:55:06.581 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-01T14:55:06.582 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2023-10-01T14:55:06.582 DEBUG:teuthology.orchestra.run.smithi031:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-01T14:55:06.643 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-01T14:55:06.643 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2023-10-01T14:55:06.644 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd if=/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-01T14:55:06.717 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-01T14:55:06.717 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2023-10-01T14:55:06.717 DEBUG:teuthology.orchestra.run.smithi031:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-01T14:55:06.779 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-01T14:55:06.779 DEBUG:teuthology.orchestra.run.smithi031:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDxbDyQLuaqT2kOd0OlfKrtcoakTzSZT4BaYaY1sFiYLrr8S91iqfF38UgBvBJknBQHJAyX2T6SggI4hGNTLPpMCMtGGmn8hY5y6iMml6I61x8zPgZVs5EF4DZguSXq4VM7SXLHy/fs6dDmJ3Ir0hS9BuyjEI1PzwEcdf9uz8Q4AH7JNjtnDsxTAxAl1ZZSp8hgUvCei9eiACxBGzQu1Nl0D9wefSxlPRV3qCXK/5p9khGZ4nITDxDn63piccD06zQq2P+xIbmPQzIt3zGltJHEXUDqSkvMv+VjDm9crb51QPsSdWPMr8psGv0mOcI0iMj8KOhMWhsw1YIv/IEIyxEiXS6fUl8CLptE1jyCyWJUcZtyXq8NsleCDTBFmcTk9mNbKnMocwBf5d9cUiaRFRc108zmz6HOtpTO70t/JpluB5ZLsue2cu/aeQ2TQ1LDgVeRX5VBMtnDzTikCpCphEpZnBldSRZu67wQIa/kh8wFsSWRzBTOr4Wk9fpNLZGF1jU= ceph-466018a4-606a-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-01T14:55:06.865 INFO:teuthology.orchestra.run.smithi031.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDxbDyQLuaqT2kOd0OlfKrtcoakTzSZT4BaYaY1sFiYLrr8S91iqfF38UgBvBJknBQHJAyX2T6SggI4hGNTLPpMCMtGGmn8hY5y6iMml6I61x8zPgZVs5EF4DZguSXq4VM7SXLHy/fs6dDmJ3Ir0hS9BuyjEI1PzwEcdf9uz8Q4AH7JNjtnDsxTAxAl1ZZSp8hgUvCei9eiACxBGzQu1Nl0D9wefSxlPRV3qCXK/5p9khGZ4nITDxDn63piccD06zQq2P+xIbmPQzIt3zGltJHEXUDqSkvMv+VjDm9crb51QPsSdWPMr8psGv0mOcI0iMj8KOhMWhsw1YIv/IEIyxEiXS6fUl8CLptE1jyCyWJUcZtyXq8NsleCDTBFmcTk9mNbKnMocwBf5d9cUiaRFRc108zmz6HOtpTO70t/JpluB5ZLsue2cu/aeQ2TQ1LDgVeRX5VBMtnDzTikCpCphEpZnBldSRZu67wQIa/kh8wFsSWRzBTOr4Wk9fpNLZGF1jU= ceph-466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:55:06.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDxbDyQLuaqT2kOd0OlfKrtcoakTzSZT4BaYaY1sFiYLrr8S91iqfF38UgBvBJknBQHJAyX2T6SggI4hGNTLPpMCMtGGmn8hY5y6iMml6I61x8zPgZVs5EF4DZguSXq4VM7SXLHy/fs6dDmJ3Ir0hS9BuyjEI1PzwEcdf9uz8Q4AH7JNjtnDsxTAxAl1ZZSp8hgUvCei9eiACxBGzQu1Nl0D9wefSxlPRV3qCXK/5p9khGZ4nITDxDn63piccD06zQq2P+xIbmPQzIt3zGltJHEXUDqSkvMv+VjDm9crb51QPsSdWPMr8psGv0mOcI0iMj8KOhMWhsw1YIv/IEIyxEiXS6fUl8CLptE1jyCyWJUcZtyXq8NsleCDTBFmcTk9mNbKnMocwBf5d9cUiaRFRc108zmz6HOtpTO70t/JpluB5ZLsue2cu/aeQ2TQ1LDgVeRX5VBMtnDzTikCpCphEpZnBldSRZu67wQIa/kh8wFsSWRzBTOr4Wk9fpNLZGF1jU= ceph-466018a4-606a-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-01T14:55:06.925 INFO:teuthology.orchestra.run.smithi099.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDxbDyQLuaqT2kOd0OlfKrtcoakTzSZT4BaYaY1sFiYLrr8S91iqfF38UgBvBJknBQHJAyX2T6SggI4hGNTLPpMCMtGGmn8hY5y6iMml6I61x8zPgZVs5EF4DZguSXq4VM7SXLHy/fs6dDmJ3Ir0hS9BuyjEI1PzwEcdf9uz8Q4AH7JNjtnDsxTAxAl1ZZSp8hgUvCei9eiACxBGzQu1Nl0D9wefSxlPRV3qCXK/5p9khGZ4nITDxDn63piccD06zQq2P+xIbmPQzIt3zGltJHEXUDqSkvMv+VjDm9crb51QPsSdWPMr8psGv0mOcI0iMj8KOhMWhsw1YIv/IEIyxEiXS6fUl8CLptE1jyCyWJUcZtyXq8NsleCDTBFmcTk9mNbKnMocwBf5d9cUiaRFRc108zmz6HOtpTO70t/JpluB5ZLsue2cu/aeQ2TQ1LDgVeRX5VBMtnDzTikCpCphEpZnBldSRZu67wQIa/kh8wFsSWRzBTOr4Wk9fpNLZGF1jU= ceph-466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:55:06.937 DEBUG:teuthology.orchestra.run.smithi135:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDxbDyQLuaqT2kOd0OlfKrtcoakTzSZT4BaYaY1sFiYLrr8S91iqfF38UgBvBJknBQHJAyX2T6SggI4hGNTLPpMCMtGGmn8hY5y6iMml6I61x8zPgZVs5EF4DZguSXq4VM7SXLHy/fs6dDmJ3Ir0hS9BuyjEI1PzwEcdf9uz8Q4AH7JNjtnDsxTAxAl1ZZSp8hgUvCei9eiACxBGzQu1Nl0D9wefSxlPRV3qCXK/5p9khGZ4nITDxDn63piccD06zQq2P+xIbmPQzIt3zGltJHEXUDqSkvMv+VjDm9crb51QPsSdWPMr8psGv0mOcI0iMj8KOhMWhsw1YIv/IEIyxEiXS6fUl8CLptE1jyCyWJUcZtyXq8NsleCDTBFmcTk9mNbKnMocwBf5d9cUiaRFRc108zmz6HOtpTO70t/JpluB5ZLsue2cu/aeQ2TQ1LDgVeRX5VBMtnDzTikCpCphEpZnBldSRZu67wQIa/kh8wFsSWRzBTOr4Wk9fpNLZGF1jU= ceph-466018a4-606a-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-01T14:55:06.984 INFO:teuthology.orchestra.run.smithi135.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDxbDyQLuaqT2kOd0OlfKrtcoakTzSZT4BaYaY1sFiYLrr8S91iqfF38UgBvBJknBQHJAyX2T6SggI4hGNTLPpMCMtGGmn8hY5y6iMml6I61x8zPgZVs5EF4DZguSXq4VM7SXLHy/fs6dDmJ3Ir0hS9BuyjEI1PzwEcdf9uz8Q4AH7JNjtnDsxTAxAl1ZZSp8hgUvCei9eiACxBGzQu1Nl0D9wefSxlPRV3qCXK/5p9khGZ4nITDxDn63piccD06zQq2P+xIbmPQzIt3zGltJHEXUDqSkvMv+VjDm9crb51QPsSdWPMr8psGv0mOcI0iMj8KOhMWhsw1YIv/IEIyxEiXS6fUl8CLptE1jyCyWJUcZtyXq8NsleCDTBFmcTk9mNbKnMocwBf5d9cUiaRFRc108zmz6HOtpTO70t/JpluB5ZLsue2cu/aeQ2TQ1LDgVeRX5VBMtnDzTikCpCphEpZnBldSRZu67wQIa/kh8wFsSWRzBTOr4Wk9fpNLZGF1jU= ceph-466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:55:06.996 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-01T14:55:07.251 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:55:07.415 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:07 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/1476754184' entity='client.admin' 2023-10-01T14:55:09.178 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-01T14:55:09.178 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-01T14:55:09.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:09 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:09.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:09 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:09.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:09 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi031", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:55:09.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:09 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:09.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:09 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:09.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:09 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:09.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:09 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/3660571137' entity='client.admin' 2023-10-01T14:55:09.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:09 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:09.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:09 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:09.592 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:55:11.164 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi099 2023-10-01T14:55:11.164 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-10-01T14:55:11.164 DEBUG:teuthology.orchestra.run.smithi099:> dd of=/etc/ceph/ceph.conf 2023-10-01T14:55:11.186 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-10-01T14:55:11.186 DEBUG:teuthology.orchestra.run.smithi099:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-01T14:55:11.247 INFO:tasks.cephadm:Adding host smithi099 to orchestrator... 2023-10-01T14:55:11.248 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi099 2023-10-01T14:55:11.504 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:55:11.957 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:11 smithi031 ceph-mon[107152]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:11.957 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:11 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:11.957 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:11 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:11.957 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:11 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:11.957 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:11 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:55:11.957 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:11 smithi031 ceph-mon[107152]: Updating smithi031:/etc/ceph/ceph.conf 2023-10-01T14:55:13.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:12 smithi031 ceph-mon[107152]: Updating smithi031:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:13.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:12 smithi031 ceph-mon[107152]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2023-10-01T14:55:13.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:12 smithi031 ceph-mon[107152]: Updating smithi031:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T14:55:13.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:13.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:13.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:14.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:13 smithi031 ceph-mon[107152]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi099", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:14.970 INFO:teuthology.orchestra.run.smithi031.stdout:Added host 'smithi099' with addr '172.21.15.99' 2023-10-01T14:55:15.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:14 smithi031 ceph-mon[107152]: Deploying cephadm binary to smithi099 2023-10-01T14:55:15.313 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-01T14:55:15.559 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:55:16.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:16.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:15 smithi031 ceph-mon[107152]: Added host smithi099 2023-10-01T14:55:16.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:16.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:16.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:16.749 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:55:16.749 INFO:teuthology.orchestra.run.smithi031.stdout:[{"addr": "172.21.15.31", "hostname": "smithi031", "labels": [], "status": ""}, {"addr": "172.21.15.99", "hostname": "smithi099", "labels": [], "status": ""}] 2023-10-01T14:55:17.184 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi135 2023-10-01T14:55:17.184 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-10-01T14:55:17.185 DEBUG:teuthology.orchestra.run.smithi135:> dd of=/etc/ceph/ceph.conf 2023-10-01T14:55:17.207 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-10-01T14:55:17.207 DEBUG:teuthology.orchestra.run.smithi135:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-01T14:55:17.267 INFO:tasks.cephadm:Adding host smithi135 to orchestrator... 2023-10-01T14:55:17.267 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi135 2023-10-01T14:55:17.520 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:55:17.647 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:17 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:17.647 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:17 smithi031 ceph-mon[107152]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T14:55:19.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:19 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:19.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:19 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:19.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:19 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:19.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:19 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:19.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:19 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi099", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:55:19.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:19 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:19.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:19 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:55:19.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:19 smithi031 ceph-mon[107152]: Updating smithi099:/etc/ceph/ceph.conf 2023-10-01T14:55:19.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:19 smithi031 ceph-mon[107152]: Updating smithi099:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:19.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:19 smithi031 ceph-mon[107152]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:19.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:19 smithi031 ceph-mon[107152]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi135", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:19.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:19 smithi031 ceph-mon[107152]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-10-01T14:55:19.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:19 smithi031 ceph-mon[107152]: Updating smithi099:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T14:55:20.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:20 smithi031 ceph-mon[107152]: Deploying cephadm binary to smithi135 2023-10-01T14:55:20.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:20 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:20.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:20 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:20.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:20 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:20.987 INFO:teuthology.orchestra.run.smithi031.stdout:Added host 'smithi135' with addr '172.21.15.135' 2023-10-01T14:55:21.403 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-01T14:55:21.652 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:55:22.176 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:21 smithi031 ceph-mon[107152]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:22.176 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:21 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:22.176 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:21 smithi031 ceph-mon[107152]: Added host smithi135 2023-10-01T14:55:22.177 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:21 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:22.177 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:21 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:22.177 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:21 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:22.905 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:55:22.905 INFO:teuthology.orchestra.run.smithi031.stdout:[{"addr": "172.21.15.31", "hostname": "smithi031", "labels": [], "status": ""}, {"addr": "172.21.15.99", "hostname": "smithi099", "labels": [], "status": ""}, {"addr": "172.21.15.135", "hostname": "smithi135", "labels": [], "status": ""}] 2023-10-01T14:55:23.371 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-01T14:55:23.371 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-01T14:55:23.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:23 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:23.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:23 smithi031 ceph-mon[107152]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:23.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:23 smithi031 ceph-mon[107152]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T14:55:23.616 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:55:25.263 INFO:teuthology.orchestra.run.smithi031.stderr:adjusted tunables profile to default 2023-10-01T14:55:25.480 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:25.480 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:25.480 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:25.481 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:25.481 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi135", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:55:25.481 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:25.481 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:55:25.481 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:25 smithi031 ceph-mon[107152]: Updating smithi135:/etc/ceph/ceph.conf 2023-10-01T14:55:25.481 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:25 smithi031 ceph-mon[107152]: Updating smithi135:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:25.481 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:25 smithi031 ceph-mon[107152]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:25.481 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:25 smithi031 ceph-mon[107152]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2023-10-01T14:55:25.481 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:25 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/77035426' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-01T14:55:25.481 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:25 smithi031 ceph-mon[107152]: Updating smithi135:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T14:55:25.617 INFO:tasks.cephadm:Adding mon.a on smithi031 2023-10-01T14:55:25.617 INFO:tasks.cephadm:Adding mon.b on smithi099 2023-10-01T14:55:25.617 INFO:tasks.cephadm:Adding mon.c on smithi135 2023-10-01T14:55:25.618 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi031:172.21.15.31=a;smithi099:172.21.15.99=b;smithi135:172.21.15.135=c' 2023-10-01T14:55:25.853 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:26.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:26 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/77035426' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-01T14:55:26.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:26 smithi031 ceph-mon[107152]: osdmap e4: 0 total, 0 up, 0 in 2023-10-01T14:55:26.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:26 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:26.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:26 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:26.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:26 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:26.939 INFO:teuthology.orchestra.run.smithi135.stdout:Scheduled mon update... 2023-10-01T14:55:27.322 DEBUG:teuthology.orchestra.run.smithi099:mon.b> sudo journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-01T14:55:27.325 DEBUG:teuthology.orchestra.run.smithi135:mon.c> sudo journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-01T14:55:27.327 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-01T14:55:27.327 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-01T14:55:27.357 INFO:journalctl@ceph.mon.c.smithi135.stdout:-- Logs begin at Sun 2023-10-01 14:44:43 UTC. -- 2023-10-01T14:55:27.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:-- Logs begin at Sun 2023-10-01 14:44:43 UTC. -- 2023-10-01T14:55:27.970 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:28.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:27 smithi031 ceph-mon[107152]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:28.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:27 smithi031 ceph-mon[107152]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi031:172.21.15.31=a;smithi099:172.21.15.99=b;smithi135:172.21.15.135=c", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:28.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:27 smithi031 ceph-mon[107152]: Saving service mon spec with placement smithi031:172.21.15.31=a;smithi099:172.21.15.99=b;smithi135:172.21.15.135=c;count:3 2023-10-01T14:55:28.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:28.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:28.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:28.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:55:28.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:28.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:28.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:28.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:27 smithi031 ceph-mon[107152]: Deploying daemon mon.c on smithi135 2023-10-01T14:55:29.577 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:55:29.577 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":1,"fsid":"466018a4-606a-11ee-8db5-212e2dc638e7","modified":"2023-10-01T14:54:09.126194Z","created":"2023-10-01T14:54:09.126194Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:3300","nonce":0},{"type":"v1","addr":"172.21.15.31:6789","nonce":0}]},"addr":"172.21.15.31:6789/0","public_addr":"172.21.15.31:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-01T14:55:29.579 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 1 2023-10-01T14:55:29.599 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 systemd[1]: Starting Ceph mon.c for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T14:55:29.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 bash[109307]: 04dde62a2fbca1305c58831e1b849265ba40aec1200fc5e3060c6fd2264e0650 2023-10-01T14:55:29.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: set uid:gid to 167:167 (ceph:ceph) 2023-10-01T14:55:29.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev), process ceph-mon, pid 2 2023-10-01T14:55:29.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: pidfile_write: ignore empty --pid-file 2023-10-01T14:55:29.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 systemd[1]: Started Ceph mon.c for 466018a4-606a-11ee-8db5-212e2dc638e7. 2023-10-01T14:55:29.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: load: jerasure load: lrc 2023-10-01T14:55:29.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: RocksDB version: 7.9.2 2023-10-01T14:55:29.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Git sha 0 2023-10-01T14:55:29.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Compile date 2023-09-25 19:01:54 2023-10-01T14:55:29.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: DB SUMMARY 2023-10-01T14:55:29.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: DB Session ID: 9BVZI4FWEKKWIG4465B3 2023-10-01T14:55:29.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: CURRENT file: CURRENT 2023-10-01T14:55:29.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: IDENTITY file: IDENTITY 2023-10-01T14:55:29.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2023-10-01T14:55:29.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-10-01T14:55:29.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2023-10-01T14:55:29.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.error_if_exists: 0 2023-10-01T14:55:29.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.create_if_missing: 0 2023-10-01T14:55:29.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.paranoid_checks: 1 2023-10-01T14:55:29.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-01T14:55:29.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-01T14:55:29.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-01T14:55:29.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.env: 0x55dd9d353da0 2023-10-01T14:55:29.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.fs: PosixFileSystem 2023-10-01T14:55:29.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.info_log: 0x55dd9dbc7460 2023-10-01T14:55:29.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_file_opening_threads: 16 2023-10-01T14:55:29.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.statistics: (nil) 2023-10-01T14:55:29.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.use_fsync: 0 2023-10-01T14:55:29.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_log_file_size: 0 2023-10-01T14:55:29.922 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T14:55:29.923 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T14:55:29.923 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.keep_log_file_num: 1000 2023-10-01T14:55:29.923 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.recycle_log_file_num: 0 2023-10-01T14:55:29.923 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.allow_fallocate: 1 2023-10-01T14:55:29.923 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.allow_mmap_reads: 0 2023-10-01T14:55:29.923 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.allow_mmap_writes: 0 2023-10-01T14:55:29.923 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.use_direct_reads: 0 2023-10-01T14:55:29.923 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T14:55:29.923 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.create_missing_column_families: 0 2023-10-01T14:55:29.923 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.db_log_dir: 2023-10-01T14:55:29.923 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.wal_dir: 2023-10-01T14:55:29.923 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T14:55:29.923 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T14:55:29.924 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T14:55:29.924 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T14:55:29.924 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T14:55:29.924 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T14:55:29.924 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.advise_random_on_open: 1 2023-10-01T14:55:29.924 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.db_write_buffer_size: 0 2023-10-01T14:55:29.924 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.write_buffer_manager: 0x55dd9db814a0 2023-10-01T14:55:29.924 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T14:55:29.925 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T14:55:29.925 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T14:55:29.925 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.rate_limiter: (nil) 2023-10-01T14:55:29.925 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T14:55:29.925 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.wal_recovery_mode: 2 2023-10-01T14:55:29.925 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.enable_thread_tracking: 0 2023-10-01T14:55:29.925 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.enable_pipelined_write: 0 2023-10-01T14:55:29.925 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.unordered_write: 0 2023-10-01T14:55:29.925 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T14:55:29.925 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T14:55:29.925 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T14:55:29.925 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T14:55:29.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.row_cache: None 2023-10-01T14:55:29.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.wal_filter: None 2023-10-01T14:55:29.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T14:55:29.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.allow_ingest_behind: 0 2023-10-01T14:55:29.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.two_write_queues: 0 2023-10-01T14:55:29.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.manual_wal_flush: 0 2023-10-01T14:55:29.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.wal_compression: 0 2023-10-01T14:55:29.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.atomic_flush: 0 2023-10-01T14:55:29.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T14:55:29.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T14:55:29.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T14:55:29.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.log_readahead_size: 0 2023-10-01T14:55:29.926 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-01T14:55:29.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.best_efforts_recovery: 0 2023-10-01T14:55:29.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-01T14:55:29.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-01T14:55:29.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.allow_data_in_errors: 0 2023-10-01T14:55:29.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.db_host_id: __hostname__ 2023-10-01T14:55:29.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-01T14:55:29.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_background_jobs: 2 2023-10-01T14:55:29.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_background_compactions: -1 2023-10-01T14:55:29.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_subcompactions: 1 2023-10-01T14:55:29.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T14:55:29.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T14:55:29.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T14:55:29.927 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_total_wal_size: 0 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_open_files: -1 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.bytes_per_sync: 0 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compaction_readahead_size: 0 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_background_flushes: -1 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Compression algorithms supported: 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: kZSTD supported: 0 2023-10-01T14:55:29.928 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: kXpressCompression supported: 0 2023-10-01T14:55:29.929 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: kLZ4HCCompression supported: 1 2023-10-01T14:55:29.929 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: kZlibCompression supported: 1 2023-10-01T14:55:29.929 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: kSnappyCompression supported: 1 2023-10-01T14:55:29.929 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: kLZ4Compression supported: 1 2023-10-01T14:55:29.929 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: kBZip2Compression supported: 0 2023-10-01T14:55:29.929 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T14:55:29.929 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-01T14:55:29.929 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-10-01T14:55:29.929 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-01T14:55:29.929 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T14:55:29.929 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.merge_operator: 2023-10-01T14:55:29.929 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compaction_filter: None 2023-10-01T14:55:29.929 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compaction_filter_factory: None 2023-10-01T14:55:29.930 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.sst_partitioner_factory: None 2023-10-01T14:55:29.930 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T14:55:29.930 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T14:55:29.930 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55dd9dbc7560) 2023-10-01T14:55:29.930 INFO:journalctl@ceph.mon.c.smithi135.stdout: cache_index_and_filter_blocks: 1 2023-10-01T14:55:29.930 INFO:journalctl@ceph.mon.c.smithi135.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T14:55:29.930 INFO:journalctl@ceph.mon.c.smithi135.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T14:55:29.930 INFO:journalctl@ceph.mon.c.smithi135.stdout: pin_top_level_index_and_filter: 1 2023-10-01T14:55:29.930 INFO:journalctl@ceph.mon.c.smithi135.stdout: index_type: 0 2023-10-01T14:55:29.930 INFO:journalctl@ceph.mon.c.smithi135.stdout: data_block_index_type: 0 2023-10-01T14:55:29.930 INFO:journalctl@ceph.mon.c.smithi135.stdout: index_shortening: 1 2023-10-01T14:55:29.930 INFO:journalctl@ceph.mon.c.smithi135.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-01T14:55:29.930 INFO:journalctl@ceph.mon.c.smithi135.stdout: checksum: 4 2023-10-01T14:55:29.931 INFO:journalctl@ceph.mon.c.smithi135.stdout: no_block_cache: 0 2023-10-01T14:55:29.931 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_cache: 0x55dd9dc1f1f0 2023-10-01T14:55:29.931 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_cache_name: BinnedLRUCache 2023-10-01T14:55:29.931 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_cache_options: 2023-10-01T14:55:29.931 INFO:journalctl@ceph.mon.c.smithi135.stdout: capacity : 536870912 2023-10-01T14:55:29.931 INFO:journalctl@ceph.mon.c.smithi135.stdout: num_shard_bits : 4 2023-10-01T14:55:29.931 INFO:journalctl@ceph.mon.c.smithi135.stdout: strict_capacity_limit : 0 2023-10-01T14:55:29.931 INFO:journalctl@ceph.mon.c.smithi135.stdout: high_pri_pool_ratio: 0.000 2023-10-01T14:55:29.931 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_cache_compressed: (nil) 2023-10-01T14:55:29.931 INFO:journalctl@ceph.mon.c.smithi135.stdout: persistent_cache: (nil) 2023-10-01T14:55:29.931 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_size: 4096 2023-10-01T14:55:29.931 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_size_deviation: 10 2023-10-01T14:55:29.931 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_restart_interval: 16 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: index_block_restart_interval: 1 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: metadata_block_size: 4096 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: partition_filters: 0 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: use_delta_encoding: 1 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: filter_policy: bloomfilter 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: whole_key_filtering: 1 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: verify_compression: 0 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: read_amp_bytes_per_bit: 0 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: format_version: 5 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: enable_index_compression: 1 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: block_align: 0 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: max_auto_readahead_size: 262144 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: prepopulate_block_cache: 0 2023-10-01T14:55:29.932 INFO:journalctl@ceph.mon.c.smithi135.stdout: initial_auto_readahead_size: 8192 2023-10-01T14:55:29.933 INFO:journalctl@ceph.mon.c.smithi135.stdout: num_file_reads_for_auto_readahead: 2 2023-10-01T14:55:29.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.write_buffer_size: 33554432 2023-10-01T14:55:29.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_write_buffer_number: 2 2023-10-01T14:55:29.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compression: NoCompression 2023-10-01T14:55:29.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.bottommost_compression: Disabled 2023-10-01T14:55:29.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.prefix_extractor: nullptr 2023-10-01T14:55:29.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T14:55:29.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.num_levels: 7 2023-10-01T14:55:29.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T14:55:29.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T14:55:29.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T14:55:29.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T14:55:29.933 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T14:55:29.934 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T14:55:29.935 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T14:55:29.935 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T14:55:29.935 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-01T14:55:29.935 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T14:55:29.935 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-01T14:55:29.935 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-01T14:55:29.935 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T14:55:29.935 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compression_opts.level: 32767 2023-10-01T14:55:29.935 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compression_opts.strategy: 0 2023-10-01T14:55:29.935 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T14:55:29.935 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T14:55:29.935 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-01T14:55:29.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-01T14:55:29.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compression_opts.enabled: false 2023-10-01T14:55:29.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-01T14:55:29.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T14:55:29.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T14:55:29.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T14:55:29.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.target_file_size_base: 67108864 2023-10-01T14:55:29.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T14:55:29.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T14:55:29.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T14:55:29.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T14:55:29.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T14:55:29.936 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T14:55:29.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T14:55:29.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T14:55:29.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T14:55:29.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T14:55:29.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T14:55:29.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T14:55:29.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T14:55:29.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-01T14:55:29.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.arena_block_size: 1048576 2023-10-01T14:55:29.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T14:55:29.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T14:55:29.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.disable_auto_compactions: 0 2023-10-01T14:55:29.937 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T14:55:29.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T14:55:29.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T14:55:29.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T14:55:29.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T14:55:29.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T14:55:29.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T14:55:29.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T14:55:29.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T14:55:29.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T14:55:29.938 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.table_properties_collectors: 2023-10-01T14:55:29.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.inplace_update_support: 0 2023-10-01T14:55:29.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T14:55:29.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T14:55:29.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T14:55:29.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T14:55:29.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.bloom_locality: 0 2023-10-01T14:55:29.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.max_successive_merges: 0 2023-10-01T14:55:29.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T14:55:29.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.paranoid_file_checks: 0 2023-10-01T14:55:29.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.force_consistency_checks: 1 2023-10-01T14:55:29.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.report_bg_io_stats: 0 2023-10-01T14:55:29.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.ttl: 2592000 2023-10-01T14:55:29.939 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T14:55:29.940 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-01T14:55:29.940 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-01T14:55:29.940 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.enable_blob_files: false 2023-10-01T14:55:29.940 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.min_blob_size: 0 2023-10-01T14:55:29.940 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.blob_file_size: 268435456 2023-10-01T14:55:29.940 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-01T14:55:29.940 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-01T14:55:29.940 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-01T14:55:29.940 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-01T14:55:29.940 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-01T14:55:29.940 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.blob_file_starting_level: 0 2023-10-01T14:55:29.940 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-01T14:55:29.941 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T14:55:29.941 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2023-10-01T14:55:29.941 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 997e6a2b-909b-42c3-8930-a13e29c37df4 2023-10-01T14:55:29.941 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696172129690627, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-10-01T14:55:29.941 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2023-10-01T14:55:29.941 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696172129691015, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1622, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696172129, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "997e6a2b-909b-42c3-8930-a13e29c37df4", "db_session_id": "9BVZI4FWEKKWIG4465B3", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2023-10-01T14:55:29.941 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696172129691103, "job": 1, "event": "recovery_finished"} 2023-10-01T14:55:29.941 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2023-10-01T14:55:29.941 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-01T14:55:29.941 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55dd9dcbc000 2023-10-01T14:55:29.941 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: DB pointer 0x55dd9dca8000 2023-10-01T14:55:29.942 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-01T14:55:29.942 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-01T14:55:29.942 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** DB Stats ** 2023-10-01T14:55:29.942 INFO:journalctl@ceph.mon.c.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T14:55:29.942 INFO:journalctl@ceph.mon.c.smithi135.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T14:55:29.942 INFO:journalctl@ceph.mon.c.smithi135.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T14:55:29.942 INFO:journalctl@ceph.mon.c.smithi135.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T14:55:29.942 INFO:journalctl@ceph.mon.c.smithi135.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T14:55:29.942 INFO:journalctl@ceph.mon.c.smithi135.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T14:55:29.942 INFO:journalctl@ceph.mon.c.smithi135.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T14:55:29.942 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-10-01T14:55:29.942 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** Compaction Stats [default] ** 2023-10-01T14:55:29.942 INFO:journalctl@ceph.mon.c.smithi135.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T14:55:29.943 INFO:journalctl@ceph.mon.c.smithi135.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-01T14:55:29.943 INFO:journalctl@ceph.mon.c.smithi135.stdout: L0 1/0 1.58 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-10-01T14:55:29.943 INFO:journalctl@ceph.mon.c.smithi135.stdout: Sum 1/0 1.58 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-10-01T14:55:29.943 INFO:journalctl@ceph.mon.c.smithi135.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-10-01T14:55:29.943 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-10-01T14:55:29.943 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** Compaction Stats [default] ** 2023-10-01T14:55:29.943 INFO:journalctl@ceph.mon.c.smithi135.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T14:55:29.943 INFO:journalctl@ceph.mon.c.smithi135.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T14:55:29.943 INFO:journalctl@ceph.mon.c.smithi135.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.1 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-10-01T14:55:29.943 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-10-01T14:55:29.943 INFO:journalctl@ceph.mon.c.smithi135.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-01T14:55:29.943 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-10-01T14:55:29.944 INFO:journalctl@ceph.mon.c.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T14:55:29.944 INFO:journalctl@ceph.mon.c.smithi135.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T14:55:29.944 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T14:55:29.944 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T14:55:29.944 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T14:55:29.944 INFO:journalctl@ceph.mon.c.smithi135.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-01T14:55:29.944 INFO:journalctl@ceph.mon.c.smithi135.stdout: Cumulative compaction: 0.00 GB write, 0.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T14:55:29.944 INFO:journalctl@ceph.mon.c.smithi135.stdout: Interval compaction: 0.00 GB write, 0.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T14:55:29.944 INFO:journalctl@ceph.mon.c.smithi135.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T14:55:29.944 INFO:journalctl@ceph.mon.c.smithi135.stdout: Block cache BinnedLRUCache@0x55dd9dc1f1f0#2 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-10-01T14:55:29.944 INFO:journalctl@ceph.mon.c.smithi135.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2023-10-01T14:55:29.944 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-10-01T14:55:29.944 INFO:journalctl@ceph.mon.c.smithi135.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-01T14:55:29.945 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-10-01T14:55:29.945 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: using public_addr v2:172.21.15.135:0/0 -> [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] 2023-10-01T14:55:29.945 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: starting mon.c rank -1 at public addrs [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] at bind addrs [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] mon_data /var/lib/ceph/mon/ceph-c fsid 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:55:29.945 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(???) e0 preinit fsid 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:55:29.945 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing).mds e1 new map 2023-10-01T14:55:29.945 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing).mds e1 print_map 2023-10-01T14:55:29.945 INFO:journalctl@ceph.mon.c.smithi135.stdout: e1 2023-10-01T14:55:29.945 INFO:journalctl@ceph.mon.c.smithi135.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T14:55:29.945 INFO:journalctl@ceph.mon.c.smithi135.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments} 2023-10-01T14:55:29.945 INFO:journalctl@ceph.mon.c.smithi135.stdout: legacy client fscid: -1 2023-10-01T14:55:29.945 INFO:journalctl@ceph.mon.c.smithi135.stdout: 2023-10-01T14:55:29.945 INFO:journalctl@ceph.mon.c.smithi135.stdout: No filesystems configured 2023-10-01T14:55:29.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-01T14:55:29.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-01T14:55:29.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-01T14:55:29.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-01T14:55:29.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-01T14:55:29.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-01T14:55:29.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-01T14:55:29.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T14:55:29.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T14:55:29.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T14:55:29.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/77035426' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-01T14:55:29.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: osdmap e4: 0 total, 0 up, 0 in 2023-10-01T14:55:29.946 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:29.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:29.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:29.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:29.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi031:172.21.15.31=a;smithi099:172.21.15.99=b;smithi135:172.21.15.135=c", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:29.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: Saving service mon spec with placement smithi031:172.21.15.31=a;smithi099:172.21.15.99=b;smithi135:172.21.15.135=c;count:3 2023-10-01T14:55:29.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:29.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:29.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:29.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:55:29.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:29.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:29.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:29.947 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: Deploying daemon mon.c on smithi135 2023-10-01T14:55:29.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-01T14:55:29.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-10-01T14:55:29.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: expand_channel_meta expand map: {default=false} 2023-10-01T14:55:29.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: expand_channel_meta from 'false' to 'false' 2023-10-01T14:55:29.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: expand_channel_meta expanded map: {default=false} 2023-10-01T14:55:29.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: expand_channel_meta expand map: {default=info} 2023-10-01T14:55:29.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: expand_channel_meta from 'info' to 'info' 2023-10-01T14:55:29.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: expand_channel_meta expanded map: {default=info} 2023-10-01T14:55:29.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: expand_channel_meta expand map: {default=daemon} 2023-10-01T14:55:29.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-01T14:55:29.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: expand_channel_meta expanded map: {default=daemon} 2023-10-01T14:55:29.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: expand_channel_meta expand map: {default=debug} 2023-10-01T14:55:29.948 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: expand_channel_meta from 'debug' to 'debug' 2023-10-01T14:55:29.949 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:29 smithi135 ceph-mon[109341]: expand_channel_meta expanded map: {default=debug} 2023-10-01T14:55:31.069 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-01T14:55:31.070 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-01T14:55:31.299 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.c/config 2023-10-01T14:55:35.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:55:35.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: mon.a calling monitor election 2023-10-01T14:55:35.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:35.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:35.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:35.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:35.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: mon.c calling monitor election 2023-10-01T14:55:35.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:35.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:35.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:35.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:35.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-10-01T14:55:35.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: monmap e2: 2 mons at {a=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],c=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0]} removed_ranks: {} 2023-10-01T14:55:35.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: fsmap 2023-10-01T14:55:35.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: osdmap e4: 0 total, 0 up, 0 in 2023-10-01T14:55:35.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: mgrmap e13: x(active, since 36s) 2023-10-01T14:55:35.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: overall HEALTH_OK 2023-10-01T14:55:35.172 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:35.172 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:35.172 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:35.172 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:35.172 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:34 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:35.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:55:35.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: mon.a calling monitor election 2023-10-01T14:55:35.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:35.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:35.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:35.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:35.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: mon.c calling monitor election 2023-10-01T14:55:35.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:35.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:35.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:35.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:35.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-10-01T14:55:35.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: monmap e2: 2 mons at {a=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],c=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0]} removed_ranks: {} 2023-10-01T14:55:35.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: fsmap 2023-10-01T14:55:35.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: osdmap e4: 0 total, 0 up, 0 in 2023-10-01T14:55:35.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: mgrmap e13: x(active, since 36s) 2023-10-01T14:55:35.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: overall HEALTH_OK 2023-10-01T14:55:35.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:35.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:35.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:35.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:35.247 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:34 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:35.451 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:55:35.451 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":2,"fsid":"466018a4-606a-11ee-8db5-212e2dc638e7","modified":"2023-10-01T14:55:29.716868Z","created":"2023-10-01T14:54:09.126194Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:3300","nonce":0},{"type":"v1","addr":"172.21.15.31:6789","nonce":0}]},"addr":"172.21.15.31:6789/0","public_addr":"172.21.15.31:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:3300","nonce":0},{"type":"v1","addr":"172.21.15.135:6789","nonce":0}]},"addr":"172.21.15.135:6789/0","public_addr":"172.21.15.135:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-01T14:55:35.454 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 2 2023-10-01T14:55:36.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:35 smithi135 ceph-mon[109341]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:36.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:35 smithi135 ceph-mon[109341]: Deploying daemon mon.b on smithi099 2023-10-01T14:55:36.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:35 smithi135 ceph-mon[109341]: from='client.? 172.21.15.135:0/2086186259' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T14:55:36.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:35 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:36.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:35 smithi031 ceph-mon[107152]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:36.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:35 smithi031 ceph-mon[107152]: Deploying daemon mon.b on smithi099 2023-10-01T14:55:36.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:35 smithi031 ceph-mon[107152]: from='client.? 172.21.15.135:0/2086186259' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T14:55:36.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:35 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:36.806 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-01T14:55:36.807 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-01T14:55:37.034 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.c/config 2023-10-01T14:55:37.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 systemd[1]: Started Ceph mon.b for 466018a4-606a-11ee-8db5-212e2dc638e7. 2023-10-01T14:55:37.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: load: jerasure load: lrc 2023-10-01T14:55:37.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: RocksDB version: 7.9.2 2023-10-01T14:55:37.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Git sha 0 2023-10-01T14:55:37.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Compile date 2023-09-25 19:01:54 2023-10-01T14:55:37.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: DB SUMMARY 2023-10-01T14:55:37.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: DB Session ID: 4SGETSKGI1W5S7J1RVN5 2023-10-01T14:55:37.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: CURRENT file: CURRENT 2023-10-01T14:55:37.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: IDENTITY file: IDENTITY 2023-10-01T14:55:37.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2023-10-01T14:55:37.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-10-01T14:55:37.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2023-10-01T14:55:37.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.error_if_exists: 0 2023-10-01T14:55:37.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.create_if_missing: 0 2023-10-01T14:55:37.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.paranoid_checks: 1 2023-10-01T14:55:37.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-01T14:55:37.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-01T14:55:37.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-01T14:55:37.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.env: 0x55835e5a4da0 2023-10-01T14:55:37.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.fs: PosixFileSystem 2023-10-01T14:55:37.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.info_log: 0x55835fc49460 2023-10-01T14:55:37.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_file_opening_threads: 16 2023-10-01T14:55:37.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.statistics: (nil) 2023-10-01T14:55:37.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.use_fsync: 0 2023-10-01T14:55:37.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_log_file_size: 0 2023-10-01T14:55:37.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T14:55:37.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T14:55:37.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.keep_log_file_num: 1000 2023-10-01T14:55:37.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.recycle_log_file_num: 0 2023-10-01T14:55:37.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.allow_fallocate: 1 2023-10-01T14:55:37.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.allow_mmap_reads: 0 2023-10-01T14:55:37.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.allow_mmap_writes: 0 2023-10-01T14:55:37.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.use_direct_reads: 0 2023-10-01T14:55:37.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T14:55:37.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.create_missing_column_families: 0 2023-10-01T14:55:37.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.db_log_dir: 2023-10-01T14:55:37.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.wal_dir: 2023-10-01T14:55:37.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T14:55:37.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T14:55:37.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T14:55:37.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T14:55:37.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T14:55:37.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T14:55:37.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.advise_random_on_open: 1 2023-10-01T14:55:37.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.db_write_buffer_size: 0 2023-10-01T14:55:37.344 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.write_buffer_manager: 0x55835fc034a0 2023-10-01T14:55:37.344 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T14:55:37.344 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T14:55:37.344 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T14:55:37.344 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.rate_limiter: (nil) 2023-10-01T14:55:37.344 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T14:55:37.344 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.wal_recovery_mode: 2 2023-10-01T14:55:37.344 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.enable_thread_tracking: 0 2023-10-01T14:55:37.344 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.enable_pipelined_write: 0 2023-10-01T14:55:37.344 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.unordered_write: 0 2023-10-01T14:55:37.344 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T14:55:37.344 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T14:55:37.344 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T14:55:37.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T14:55:37.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.row_cache: None 2023-10-01T14:55:37.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.wal_filter: None 2023-10-01T14:55:37.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T14:55:37.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.allow_ingest_behind: 0 2023-10-01T14:55:37.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.two_write_queues: 0 2023-10-01T14:55:37.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.manual_wal_flush: 0 2023-10-01T14:55:37.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.wal_compression: 0 2023-10-01T14:55:37.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.atomic_flush: 0 2023-10-01T14:55:37.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T14:55:37.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T14:55:37.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T14:55:37.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.log_readahead_size: 0 2023-10-01T14:55:37.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-01T14:55:37.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.best_efforts_recovery: 0 2023-10-01T14:55:37.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-01T14:55:37.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-01T14:55:37.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.allow_data_in_errors: 0 2023-10-01T14:55:37.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.db_host_id: __hostname__ 2023-10-01T14:55:37.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-01T14:55:37.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_background_jobs: 2 2023-10-01T14:55:37.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_background_compactions: -1 2023-10-01T14:55:37.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_subcompactions: 1 2023-10-01T14:55:37.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T14:55:37.347 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T14:55:37.347 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T14:55:37.347 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_total_wal_size: 0 2023-10-01T14:55:37.347 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T14:55:37.347 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T14:55:37.347 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T14:55:37.347 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T14:55:37.347 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_open_files: -1 2023-10-01T14:55:37.347 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.bytes_per_sync: 0 2023-10-01T14:55:37.347 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T14:55:37.347 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T14:55:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compaction_readahead_size: 0 2023-10-01T14:55:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_background_flushes: -1 2023-10-01T14:55:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Compression algorithms supported: 2023-10-01T14:55:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T14:55:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: kZSTD supported: 0 2023-10-01T14:55:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: kXpressCompression supported: 0 2023-10-01T14:55:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: kLZ4HCCompression supported: 1 2023-10-01T14:55:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: kZlibCompression supported: 1 2023-10-01T14:55:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: kSnappyCompression supported: 1 2023-10-01T14:55:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: kLZ4Compression supported: 1 2023-10-01T14:55:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: kBZip2Compression supported: 0 2023-10-01T14:55:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T14:55:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-01T14:55:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-10-01T14:55:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-01T14:55:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T14:55:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.merge_operator: 2023-10-01T14:55:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compaction_filter: None 2023-10-01T14:55:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compaction_filter_factory: None 2023-10-01T14:55:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.sst_partitioner_factory: None 2023-10-01T14:55:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T14:55:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T14:55:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55835fc49560) 2023-10-01T14:55:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout: cache_index_and_filter_blocks: 1 2023-10-01T14:55:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T14:55:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T14:55:37.350 INFO:journalctl@ceph.mon.b.smithi099.stdout: pin_top_level_index_and_filter: 1 2023-10-01T14:55:37.350 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_type: 0 2023-10-01T14:55:37.350 INFO:journalctl@ceph.mon.b.smithi099.stdout: data_block_index_type: 0 2023-10-01T14:55:37.350 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_shortening: 1 2023-10-01T14:55:37.350 INFO:journalctl@ceph.mon.b.smithi099.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-01T14:55:37.350 INFO:journalctl@ceph.mon.b.smithi099.stdout: checksum: 4 2023-10-01T14:55:37.350 INFO:journalctl@ceph.mon.b.smithi099.stdout: no_block_cache: 0 2023-10-01T14:55:37.350 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache: 0x55835fca11f0 2023-10-01T14:55:37.350 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_name: BinnedLRUCache 2023-10-01T14:55:37.350 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_options: 2023-10-01T14:55:37.350 INFO:journalctl@ceph.mon.b.smithi099.stdout: capacity : 536870912 2023-10-01T14:55:37.350 INFO:journalctl@ceph.mon.b.smithi099.stdout: num_shard_bits : 4 2023-10-01T14:55:37.351 INFO:journalctl@ceph.mon.b.smithi099.stdout: strict_capacity_limit : 0 2023-10-01T14:55:37.351 INFO:journalctl@ceph.mon.b.smithi099.stdout: high_pri_pool_ratio: 0.000 2023-10-01T14:55:37.351 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_compressed: (nil) 2023-10-01T14:55:37.351 INFO:journalctl@ceph.mon.b.smithi099.stdout: persistent_cache: (nil) 2023-10-01T14:55:37.351 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_size: 4096 2023-10-01T14:55:37.351 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_size_deviation: 10 2023-10-01T14:55:37.351 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_restart_interval: 16 2023-10-01T14:55:37.351 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_block_restart_interval: 1 2023-10-01T14:55:37.351 INFO:journalctl@ceph.mon.b.smithi099.stdout: metadata_block_size: 4096 2023-10-01T14:55:37.351 INFO:journalctl@ceph.mon.b.smithi099.stdout: partition_filters: 0 2023-10-01T14:55:37.351 INFO:journalctl@ceph.mon.b.smithi099.stdout: use_delta_encoding: 1 2023-10-01T14:55:37.351 INFO:journalctl@ceph.mon.b.smithi099.stdout: filter_policy: bloomfilter 2023-10-01T14:55:37.351 INFO:journalctl@ceph.mon.b.smithi099.stdout: whole_key_filtering: 1 2023-10-01T14:55:37.352 INFO:journalctl@ceph.mon.b.smithi099.stdout: verify_compression: 0 2023-10-01T14:55:37.352 INFO:journalctl@ceph.mon.b.smithi099.stdout: read_amp_bytes_per_bit: 0 2023-10-01T14:55:37.352 INFO:journalctl@ceph.mon.b.smithi099.stdout: format_version: 5 2023-10-01T14:55:37.352 INFO:journalctl@ceph.mon.b.smithi099.stdout: enable_index_compression: 1 2023-10-01T14:55:37.352 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_align: 0 2023-10-01T14:55:37.352 INFO:journalctl@ceph.mon.b.smithi099.stdout: max_auto_readahead_size: 262144 2023-10-01T14:55:37.352 INFO:journalctl@ceph.mon.b.smithi099.stdout: prepopulate_block_cache: 0 2023-10-01T14:55:37.352 INFO:journalctl@ceph.mon.b.smithi099.stdout: initial_auto_readahead_size: 8192 2023-10-01T14:55:37.352 INFO:journalctl@ceph.mon.b.smithi099.stdout: num_file_reads_for_auto_readahead: 2 2023-10-01T14:55:37.352 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.write_buffer_size: 33554432 2023-10-01T14:55:37.352 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_write_buffer_number: 2 2023-10-01T14:55:37.352 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compression: NoCompression 2023-10-01T14:55:37.352 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.bottommost_compression: Disabled 2023-10-01T14:55:37.353 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.prefix_extractor: nullptr 2023-10-01T14:55:37.353 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T14:55:37.353 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.num_levels: 7 2023-10-01T14:55:37.353 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T14:55:37.353 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T14:55:37.353 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T14:55:37.353 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T14:55:37.353 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T14:55:37.353 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T14:55:37.353 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T14:55:37.353 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T14:55:37.353 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-01T14:55:37.355 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T14:55:37.355 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-01T14:55:37.355 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-01T14:55:37.355 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T14:55:37.355 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compression_opts.level: 32767 2023-10-01T14:55:37.355 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compression_opts.strategy: 0 2023-10-01T14:55:37.355 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T14:55:37.355 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T14:55:37.355 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-01T14:55:37.355 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-01T14:55:37.355 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compression_opts.enabled: false 2023-10-01T14:55:37.355 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-01T14:55:37.356 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T14:55:37.356 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T14:55:37.356 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T14:55:37.356 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.target_file_size_base: 67108864 2023-10-01T14:55:37.356 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T14:55:37.356 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T14:55:37.356 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T14:55:37.356 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T14:55:37.356 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T14:55:37.356 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T14:55:37.356 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T14:55:37.356 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T14:55:37.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T14:55:37.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T14:55:37.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T14:55:37.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T14:55:37.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T14:55:37.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-01T14:55:37.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.arena_block_size: 1048576 2023-10-01T14:55:37.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T14:55:37.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T14:55:37.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.disable_auto_compactions: 0 2023-10-01T14:55:37.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T14:55:37.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T14:55:37.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T14:55:37.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T14:55:37.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T14:55:37.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T14:55:37.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T14:55:37.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T14:55:37.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T14:55:37.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T14:55:37.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.table_properties_collectors: 2023-10-01T14:55:37.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.inplace_update_support: 0 2023-10-01T14:55:37.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T14:55:37.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T14:55:37.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T14:55:37.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T14:55:37.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.bloom_locality: 0 2023-10-01T14:55:37.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.max_successive_merges: 0 2023-10-01T14:55:37.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T14:55:37.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.paranoid_file_checks: 0 2023-10-01T14:55:37.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.force_consistency_checks: 1 2023-10-01T14:55:37.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.report_bg_io_stats: 0 2023-10-01T14:55:37.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.ttl: 2592000 2023-10-01T14:55:37.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T14:55:37.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-01T14:55:37.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-01T14:55:37.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.enable_blob_files: false 2023-10-01T14:55:37.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.min_blob_size: 0 2023-10-01T14:55:37.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.blob_file_size: 268435456 2023-10-01T14:55:37.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-01T14:55:37.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-01T14:55:37.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-01T14:55:37.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-01T14:55:37.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-01T14:55:37.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.blob_file_starting_level: 0 2023-10-01T14:55:37.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-01T14:55:37.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T14:55:37.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2023-10-01T14:55:37.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 90ad3f8c-883d-4ead-8745-a59a09bc976f 2023-10-01T14:55:37.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696172137011120, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-10-01T14:55:37.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2023-10-01T14:55:37.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696172137011671, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1747, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696172137, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "90ad3f8c-883d-4ead-8745-a59a09bc976f", "db_session_id": "4SGETSKGI1W5S7J1RVN5", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2023-10-01T14:55:37.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696172137011764, "job": 1, "event": "recovery_finished"} 2023-10-01T14:55:37.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2023-10-01T14:55:37.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-01T14:55:37.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55835fd3e000 2023-10-01T14:55:37.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: DB pointer 0x55835fd2a000 2023-10-01T14:55:37.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-01T14:55:37.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-01T14:55:37.361 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** DB Stats ** 2023-10-01T14:55:37.361 INFO:journalctl@ceph.mon.b.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T14:55:37.361 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T14:55:37.361 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T14:55:37.362 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T14:55:37.362 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T14:55:37.362 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T14:55:37.362 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T14:55:37.362 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-10-01T14:55:37.362 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2023-10-01T14:55:37.362 INFO:journalctl@ceph.mon.b.smithi099.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T14:55:37.362 INFO:journalctl@ceph.mon.b.smithi099.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-01T14:55:37.362 INFO:journalctl@ceph.mon.b.smithi099.stdout: L0 1/0 1.71 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-01T14:55:37.362 INFO:journalctl@ceph.mon.b.smithi099.stdout: Sum 1/0 1.71 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-01T14:55:37.362 INFO:journalctl@ceph.mon.b.smithi099.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-01T14:55:37.362 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-10-01T14:55:37.363 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2023-10-01T14:55:37.363 INFO:journalctl@ceph.mon.b.smithi099.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T14:55:37.363 INFO:journalctl@ceph.mon.b.smithi099.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T14:55:37.363 INFO:journalctl@ceph.mon.b.smithi099.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-01T14:55:37.363 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-10-01T14:55:37.363 INFO:journalctl@ceph.mon.b.smithi099.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-01T14:55:37.363 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-10-01T14:55:37.363 INFO:journalctl@ceph.mon.b.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T14:55:37.363 INFO:journalctl@ceph.mon.b.smithi099.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T14:55:37.363 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T14:55:37.363 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T14:55:37.363 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T14:55:37.363 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-01T14:55:37.364 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T14:55:37.364 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T14:55:37.364 INFO:journalctl@ceph.mon.b.smithi099.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T14:55:37.364 INFO:journalctl@ceph.mon.b.smithi099.stdout: Block cache BinnedLRUCache@0x55835fca11f0#2 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2023-10-01T14:55:37.364 INFO:journalctl@ceph.mon.b.smithi099.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2023-10-01T14:55:37.364 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-10-01T14:55:37.364 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-01T14:55:37.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-10-01T14:55:37.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: using public_addr v2:172.21.15.99:0/0 -> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] 2023-10-01T14:55:37.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: starting mon.b rank -1 at public addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] at bind addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:55:37.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(???) e0 preinit fsid 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:55:37.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing).mds e1 new map 2023-10-01T14:55:37.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing).mds e1 print_map 2023-10-01T14:55:37.365 INFO:journalctl@ceph.mon.b.smithi099.stdout: e1 2023-10-01T14:55:37.365 INFO:journalctl@ceph.mon.b.smithi099.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T14:55:37.365 INFO:journalctl@ceph.mon.b.smithi099.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments} 2023-10-01T14:55:37.365 INFO:journalctl@ceph.mon.b.smithi099.stdout: legacy client fscid: -1 2023-10-01T14:55:37.365 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-10-01T14:55:37.365 INFO:journalctl@ceph.mon.b.smithi099.stdout: No filesystems configured 2023-10-01T14:55:37.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-01T14:55:37.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-01T14:55:37.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-01T14:55:37.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-01T14:55:37.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-01T14:55:37.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-01T14:55:37.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-01T14:55:37.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T14:55:37.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T14:55:37.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T14:55:37.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/77035426' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-01T14:55:37.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: osdmap e4: 0 total, 0 up, 0 in 2023-10-01T14:55:37.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:37.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:37.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:37.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:37.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi031:172.21.15.31=a;smithi099:172.21.15.99=b;smithi135:172.21.15.135=c", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:37.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: Saving service mon spec with placement smithi031:172.21.15.31=a;smithi099:172.21.15.99=b;smithi135:172.21.15.135=c;count:3 2023-10-01T14:55:37.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:37.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:37.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:37.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:55:37.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:37.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:37.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:37.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: Deploying daemon mon.c on smithi135 2023-10-01T14:55:37.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:55:37.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.a calling monitor election 2023-10-01T14:55:37.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:37.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:37.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:37.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:37.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.c calling monitor election 2023-10-01T14:55:37.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:37.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:37.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:37.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:37.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-10-01T14:55:37.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: monmap e2: 2 mons at {a=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],c=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0]} removed_ranks: {} 2023-10-01T14:55:37.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: fsmap 2023-10-01T14:55:37.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: osdmap e4: 0 total, 0 up, 0 in 2023-10-01T14:55:37.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mgrmap e13: x(active, since 36s) 2023-10-01T14:55:37.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: overall HEALTH_OK 2023-10-01T14:55:37.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:37.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:37.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:37.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:37.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:37.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:37.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: Deploying daemon mon.b on smithi099 2023-10-01T14:55:37.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='client.? 172.21.15.135:0/2086186259' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T14:55:37.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:37.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-01T14:55:37.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-10-01T14:55:37.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: expand_channel_meta expand map: {default=false} 2023-10-01T14:55:37.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: expand_channel_meta from 'false' to 'false' 2023-10-01T14:55:37.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: expand_channel_meta expanded map: {default=false} 2023-10-01T14:55:37.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: expand_channel_meta expand map: {default=info} 2023-10-01T14:55:37.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: expand_channel_meta from 'info' to 'info' 2023-10-01T14:55:37.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: expand_channel_meta expanded map: {default=info} 2023-10-01T14:55:37.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: expand_channel_meta expand map: {default=daemon} 2023-10-01T14:55:37.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-01T14:55:37.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: expand_channel_meta expanded map: {default=daemon} 2023-10-01T14:55:37.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: expand_channel_meta expand map: {default=debug} 2023-10-01T14:55:37.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: expand_channel_meta from 'debug' to 'debug' 2023-10-01T14:55:37.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:37 smithi099 ceph-mon[109132]: expand_channel_meta expanded map: {default=debug} 2023-10-01T14:55:42.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:42.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:55:42.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: mon.a calling monitor election 2023-10-01T14:55:42.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: mon.c calling monitor election 2023-10-01T14:55:42.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:42.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:42.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: mon.b calling monitor election 2023-10-01T14:55:42.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:42.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2023-10-01T14:55:42.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: monmap e3: 3 mons at {a=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0]} removed_ranks: {} 2023-10-01T14:55:42.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: fsmap 2023-10-01T14:55:42.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: osdmap e4: 0 total, 0 up, 0 in 2023-10-01T14:55:42.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: mgrmap e13: x(active, since 43s) 2023-10-01T14:55:42.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: overall HEALTH_OK 2023-10-01T14:55:42.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:42.421 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:42.422 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:42.422 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:42 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:42.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:42.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:55:42.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: mon.a calling monitor election 2023-10-01T14:55:42.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: mon.c calling monitor election 2023-10-01T14:55:42.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:42.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:42.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: mon.b calling monitor election 2023-10-01T14:55:42.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:42.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2023-10-01T14:55:42.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: monmap e3: 3 mons at {a=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0]} removed_ranks: {} 2023-10-01T14:55:42.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: fsmap 2023-10-01T14:55:42.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: osdmap e4: 0 total, 0 up, 0 in 2023-10-01T14:55:42.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: mgrmap e13: x(active, since 43s) 2023-10-01T14:55:42.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: overall HEALTH_OK 2023-10-01T14:55:42.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:42.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:42.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:42.497 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:42 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:42.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:42.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:55:42.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: mon.a calling monitor election 2023-10-01T14:55:42.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: mon.c calling monitor election 2023-10-01T14:55:42.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:55:42.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:42.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: mon.b calling monitor election 2023-10-01T14:55:42.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:42.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:42.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2023-10-01T14:55:42.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: monmap e3: 3 mons at {a=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0]} removed_ranks: {} 2023-10-01T14:55:42.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: fsmap 2023-10-01T14:55:42.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: osdmap e4: 0 total, 0 up, 0 in 2023-10-01T14:55:42.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: mgrmap e13: x(active, since 43s) 2023-10-01T14:55:42.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: overall HEALTH_OK 2023-10-01T14:55:42.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:42.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:42.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:42.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:42 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:42.610 INFO:teuthology.orchestra.run.smithi135.stdout: 2023-10-01T14:55:42.611 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":3,"fsid":"466018a4-606a-11ee-8db5-212e2dc638e7","modified":"2023-10-01T14:55:37.038716Z","created":"2023-10-01T14:54:09.126194Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:3300","nonce":0},{"type":"v1","addr":"172.21.15.31:6789","nonce":0}]},"addr":"172.21.15.31:6789/0","public_addr":"172.21.15.31:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:3300","nonce":0},{"type":"v1","addr":"172.21.15.135:6789","nonce":0}]},"addr":"172.21.15.135:6789/0","public_addr":"172.21.15.135:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:3300","nonce":0},{"type":"v1","addr":"172.21.15.99:6789","nonce":0}]},"addr":"172.21.15.99:6789/0","public_addr":"172.21.15.99:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-10-01T14:55:42.613 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 3 2023-10-01T14:55:42.991 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-01T14:55:42.991 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-01T14:55:43.241 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:55:43.369 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:43.369 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:43 smithi031 ceph-mon[107152]: from='client.? 172.21.15.135:0/940409847' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T14:55:43.369 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:43 smithi031 ceph-mon[107152]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:43.369 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:43.369 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:43.370 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:43.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:43.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:43 smithi135 ceph-mon[109341]: from='client.? 172.21.15.135:0/940409847' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T14:55:43.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:43 smithi135 ceph-mon[109341]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:43.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:43.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:43.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:43.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:43.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:43 smithi099 ceph-mon[109132]: from='client.? 172.21.15.135:0/940409847' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T14:55:43.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:43 smithi099 ceph-mon[109132]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:43.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:43.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:43.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:55:44.150 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T14:55:44.038+0000 7fe37f1df700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-01T14:55:44.554 INFO:teuthology.orchestra.run.smithi031.stdout:# minimal ceph.conf for 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:55:44.554 INFO:teuthology.orchestra.run.smithi031.stdout:[global] 2023-10-01T14:55:44.554 INFO:teuthology.orchestra.run.smithi031.stdout: fsid = 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:55:44.554 INFO:teuthology.orchestra.run.smithi031.stdout: mon_host = [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] 2023-10-01T14:55:44.579 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.579 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.579 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:44.579 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:55:44.579 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: Updating smithi031:/etc/ceph/ceph.conf 2023-10-01T14:55:44.579 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: Updating smithi099:/etc/ceph/ceph.conf 2023-10-01T14:55:44.579 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: Updating smithi135:/etc/ceph/ceph.conf 2023-10-01T14:55:44.579 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: Updating smithi031:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:44.579 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: Updating smithi135:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:44.579 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: Updating smithi099:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:44.579 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.580 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.580 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.580 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.580 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.580 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.580 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.580 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.580 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.580 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.580 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.580 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.580 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:44.581 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T14:55:44.581 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:44.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:44.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:55:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: Updating smithi031:/etc/ceph/ceph.conf 2023-10-01T14:55:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: Updating smithi099:/etc/ceph/ceph.conf 2023-10-01T14:55:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: Updating smithi135:/etc/ceph/ceph.conf 2023-10-01T14:55:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: Updating smithi031:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: Updating smithi135:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: Updating smithi099:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:44.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T14:55:44.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:44.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:44.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:55:44.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: Updating smithi031:/etc/ceph/ceph.conf 2023-10-01T14:55:44.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: Updating smithi099:/etc/ceph/ceph.conf 2023-10-01T14:55:44.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: Updating smithi135:/etc/ceph/ceph.conf 2023-10-01T14:55:44.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: Updating smithi031:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:44.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: Updating smithi135:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:44.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: Updating smithi099:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T14:55:44.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:44.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:44.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T14:55:44.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:45.389 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-01T14:55:45.389 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2023-10-01T14:55:45.389 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd of=/etc/ceph/ceph.conf 2023-10-01T14:55:45.420 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:45 smithi031 ceph-mon[107152]: Reconfiguring mon.a (unknown last config time)... 2023-10-01T14:55:45.420 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:45 smithi031 ceph-mon[107152]: Reconfiguring daemon mon.a on smithi031 2023-10-01T14:55:45.420 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:45 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/1747472352' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:45.420 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:45 smithi031 ceph-mon[107152]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:45.420 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:45 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:45.421 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:45 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:45.421 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:45 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:45.421 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:45 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T14:55:45.421 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:45 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:45.425 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2023-10-01T14:55:45.425 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-01T14:55:45.499 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-10-01T14:55:45.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.conf 2023-10-01T14:55:45.532 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-10-01T14:55:45.532 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-01T14:55:45.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:45 smithi099 ceph-mon[109132]: Reconfiguring mon.a (unknown last config time)... 2023-10-01T14:55:45.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:45 smithi099 ceph-mon[109132]: Reconfiguring daemon mon.a on smithi031 2023-10-01T14:55:45.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:45 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/1747472352' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:45.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:45 smithi099 ceph-mon[109132]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:45.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:45 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:45.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:45 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:45.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:45 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:45.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:45 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T14:55:45.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:45 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:45.600 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-10-01T14:55:45.600 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/ceph/ceph.conf 2023-10-01T14:55:45.632 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:45 smithi135 ceph-mon[109341]: Reconfiguring mon.a (unknown last config time)... 2023-10-01T14:55:45.632 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:45 smithi135 ceph-mon[109341]: Reconfiguring daemon mon.a on smithi031 2023-10-01T14:55:45.632 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:45 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/1747472352' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:45.632 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:45 smithi135 ceph-mon[109341]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:45.632 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:45 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:45.633 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:45 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:45.633 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:45 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:45.633 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:45 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T14:55:45.633 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:45 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:45.636 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-10-01T14:55:45.637 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-01T14:55:45.708 INFO:tasks.cephadm:Adding mgr.x on smithi031 2023-10-01T14:55:45.708 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '1;smithi031=x' 2023-10-01T14:55:45.963 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.c/config 2023-10-01T14:55:46.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:46 smithi099 ceph-mon[109132]: Reconfiguring mon.b (monmap changed)... 2023-10-01T14:55:46.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:46 smithi099 ceph-mon[109132]: Reconfiguring daemon mon.b on smithi099 2023-10-01T14:55:46.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:46 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:46.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:46 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:46.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:46 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:46.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:46 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T14:55:46.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:46 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:46.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:46 smithi135 ceph-mon[109341]: Reconfiguring mon.b (monmap changed)... 2023-10-01T14:55:46.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:46 smithi135 ceph-mon[109341]: Reconfiguring daemon mon.b on smithi099 2023-10-01T14:55:46.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:46 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:46.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:46 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:46.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:46 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:46.615 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:46 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T14:55:46.616 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:46 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:46.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:46 smithi031 ceph-mon[107152]: Reconfiguring mon.b (monmap changed)... 2023-10-01T14:55:46.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:46 smithi031 ceph-mon[107152]: Reconfiguring daemon mon.b on smithi099 2023-10-01T14:55:46.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:46 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:46.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:46 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:46.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:46 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T14:55:46.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:46 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T14:55:46.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:46 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:47.086 INFO:teuthology.orchestra.run.smithi135.stdout:Scheduled mgr update... 2023-10-01T14:55:47.439 INFO:tasks.cephadm:Deploying OSDs... 2023-10-01T14:55:47.439 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2023-10-01T14:55:47.439 DEBUG:teuthology.orchestra.run.smithi031:> dd if=/scratch_devs of=/dev/stdout 2023-10-01T14:55:47.463 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-01T14:55:47.463 DEBUG:teuthology.orchestra.run.smithi031:> stat /dev/vg_nvme/lv_1 2023-10-01T14:55:47.525 INFO:teuthology.orchestra.run.smithi031.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-01T14:55:47.525 INFO:teuthology.orchestra.run.smithi031.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T14:55:47.525 INFO:teuthology.orchestra.run.smithi031.stdout:Device: 6h/6d Inode: 395383 Links: 1 2023-10-01T14:55:47.525 INFO:teuthology.orchestra.run.smithi031.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T14:55:47.525 INFO:teuthology.orchestra.run.smithi031.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T14:55:47.525 INFO:teuthology.orchestra.run.smithi031.stdout:Access: 2023-10-01 14:55:09.028339115 +0000 2023-10-01T14:55:47.525 INFO:teuthology.orchestra.run.smithi031.stdout:Modify: 2023-10-01 14:52:25.151522490 +0000 2023-10-01T14:55:47.526 INFO:teuthology.orchestra.run.smithi031.stdout:Change: 2023-10-01 14:52:25.151522490 +0000 2023-10-01T14:55:47.526 INFO:teuthology.orchestra.run.smithi031.stdout: Birth: - 2023-10-01T14:55:47.526 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-01T14:55:47.597 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records in 2023-10-01T14:55:47.597 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records out 2023-10-01T14:55:47.597 INFO:teuthology.orchestra.run.smithi031.stderr:512 bytes copied, 0.000260569 s, 2.0 MB/s 2023-10-01T14:55:47.599 DEBUG:teuthology.orchestra.run.smithi031:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-01T14:55:47.662 DEBUG:teuthology.orchestra.run.smithi031:> stat /dev/vg_nvme/lv_2 2023-10-01T14:55:47.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:47 smithi135 ceph-mon[109341]: Reconfiguring mon.c (monmap changed)... 2023-10-01T14:55:47.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:47 smithi135 ceph-mon[109341]: Reconfiguring daemon mon.c on smithi135 2023-10-01T14:55:47.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:47 smithi135 ceph-mon[109341]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:47.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:47 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:47.724 INFO:teuthology.orchestra.run.smithi031.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-01T14:55:47.724 INFO:teuthology.orchestra.run.smithi031.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T14:55:47.724 INFO:teuthology.orchestra.run.smithi031.stdout:Device: 6h/6d Inode: 396470 Links: 1 2023-10-01T14:55:47.724 INFO:teuthology.orchestra.run.smithi031.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T14:55:47.725 INFO:teuthology.orchestra.run.smithi031.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T14:55:47.725 INFO:teuthology.orchestra.run.smithi031.stdout:Access: 2023-10-01 14:55:09.028339115 +0000 2023-10-01T14:55:47.725 INFO:teuthology.orchestra.run.smithi031.stdout:Modify: 2023-10-01 14:52:25.718513882 +0000 2023-10-01T14:55:47.725 INFO:teuthology.orchestra.run.smithi031.stdout:Change: 2023-10-01 14:52:25.718513882 +0000 2023-10-01T14:55:47.725 INFO:teuthology.orchestra.run.smithi031.stdout: Birth: - 2023-10-01T14:55:47.725 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-01T14:55:47.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:47 smithi031 ceph-mon[107152]: Reconfiguring mon.c (monmap changed)... 2023-10-01T14:55:47.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:47 smithi031 ceph-mon[107152]: Reconfiguring daemon mon.c on smithi135 2023-10-01T14:55:47.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:47 smithi031 ceph-mon[107152]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:47.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:47 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:47.775 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records in 2023-10-01T14:55:47.776 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records out 2023-10-01T14:55:47.776 INFO:teuthology.orchestra.run.smithi031.stderr:512 bytes copied, 0.000223074 s, 2.3 MB/s 2023-10-01T14:55:47.777 DEBUG:teuthology.orchestra.run.smithi031:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-01T14:55:47.838 DEBUG:teuthology.orchestra.run.smithi031:> stat /dev/vg_nvme/lv_3 2023-10-01T14:55:47.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:47 smithi099 ceph-mon[109132]: Reconfiguring mon.c (monmap changed)... 2023-10-01T14:55:47.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:47 smithi099 ceph-mon[109132]: Reconfiguring daemon mon.c on smithi135 2023-10-01T14:55:47.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:47 smithi099 ceph-mon[109132]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:47.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:47 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:47.898 INFO:teuthology.orchestra.run.smithi031.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-01T14:55:47.898 INFO:teuthology.orchestra.run.smithi031.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T14:55:47.898 INFO:teuthology.orchestra.run.smithi031.stdout:Device: 6h/6d Inode: 389848 Links: 1 2023-10-01T14:55:47.898 INFO:teuthology.orchestra.run.smithi031.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T14:55:47.898 INFO:teuthology.orchestra.run.smithi031.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T14:55:47.898 INFO:teuthology.orchestra.run.smithi031.stdout:Access: 2023-10-01 14:55:09.028339115 +0000 2023-10-01T14:55:47.899 INFO:teuthology.orchestra.run.smithi031.stdout:Modify: 2023-10-01 14:52:26.286505260 +0000 2023-10-01T14:55:47.899 INFO:teuthology.orchestra.run.smithi031.stdout:Change: 2023-10-01 14:52:26.286505260 +0000 2023-10-01T14:55:47.899 INFO:teuthology.orchestra.run.smithi031.stdout: Birth: - 2023-10-01T14:55:47.899 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-01T14:55:47.971 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records in 2023-10-01T14:55:47.971 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records out 2023-10-01T14:55:47.971 INFO:teuthology.orchestra.run.smithi031.stderr:512 bytes copied, 0.000217144 s, 2.4 MB/s 2023-10-01T14:55:47.972 DEBUG:teuthology.orchestra.run.smithi031:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-01T14:55:48.034 DEBUG:teuthology.orchestra.run.smithi031:> stat /dev/vg_nvme/lv_4 2023-10-01T14:55:48.098 INFO:teuthology.orchestra.run.smithi031.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-01T14:55:48.098 INFO:teuthology.orchestra.run.smithi031.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T14:55:48.098 INFO:teuthology.orchestra.run.smithi031.stdout:Device: 6h/6d Inode: 395563 Links: 1 2023-10-01T14:55:48.098 INFO:teuthology.orchestra.run.smithi031.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T14:55:48.099 INFO:teuthology.orchestra.run.smithi031.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T14:55:48.099 INFO:teuthology.orchestra.run.smithi031.stdout:Access: 2023-10-01 14:55:09.028339115 +0000 2023-10-01T14:55:48.099 INFO:teuthology.orchestra.run.smithi031.stdout:Modify: 2023-10-01 14:52:26.886496152 +0000 2023-10-01T14:55:48.099 INFO:teuthology.orchestra.run.smithi031.stdout:Change: 2023-10-01 14:52:26.886496152 +0000 2023-10-01T14:55:48.099 INFO:teuthology.orchestra.run.smithi031.stdout: Birth: - 2023-10-01T14:55:48.099 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-01T14:55:48.175 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records in 2023-10-01T14:55:48.175 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records out 2023-10-01T14:55:48.175 INFO:teuthology.orchestra.run.smithi031.stderr:512 bytes copied, 0.000232988 s, 2.2 MB/s 2023-10-01T14:55:48.176 DEBUG:teuthology.orchestra.run.smithi031:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-01T14:55:48.242 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-10-01T14:55:48.242 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/scratch_devs of=/dev/stdout 2023-10-01T14:55:48.265 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-01T14:55:48.265 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_1 2023-10-01T14:55:48.327 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-01T14:55:48.327 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T14:55:48.327 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 6h/6d Inode: 395433 Links: 1 2023-10-01T14:55:48.327 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T14:55:48.327 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T14:55:48.327 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2023-10-01 14:55:36.599965093 +0000 2023-10-01T14:55:48.327 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2023-10-01 14:51:42.904393789 +0000 2023-10-01T14:55:48.327 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2023-10-01 14:51:42.904393789 +0000 2023-10-01T14:55:48.327 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: - 2023-10-01T14:55:48.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-01T14:55:48.400 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2023-10-01T14:55:48.400 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2023-10-01T14:55:48.400 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.00026678 s, 1.9 MB/s 2023-10-01T14:55:48.402 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-01T14:55:48.464 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_2 2023-10-01T14:55:48.524 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-01T14:55:48.524 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T14:55:48.524 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 6h/6d Inode: 394389 Links: 1 2023-10-01T14:55:48.524 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T14:55:48.524 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T14:55:48.524 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2023-10-01 14:55:36.599965093 +0000 2023-10-01T14:55:48.524 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2023-10-01 14:51:43.403382876 +0000 2023-10-01T14:55:48.524 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2023-10-01 14:51:43.403382876 +0000 2023-10-01T14:55:48.525 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: - 2023-10-01T14:55:48.525 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-01T14:55:48.593 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2023-10-01T14:55:48.594 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2023-10-01T14:55:48.594 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.00022396 s, 2.3 MB/s 2023-10-01T14:55:48.595 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-01T14:55:48.655 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_3 2023-10-01T14:55:48.714 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-01T14:55:48.714 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T14:55:48.714 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 6h/6d Inode: 386892 Links: 1 2023-10-01T14:55:48.714 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T14:55:48.714 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T14:55:48.714 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2023-10-01 14:55:36.599965093 +0000 2023-10-01T14:55:48.714 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2023-10-01 14:51:43.917371635 +0000 2023-10-01T14:55:48.714 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2023-10-01 14:51:43.917371635 +0000 2023-10-01T14:55:48.714 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: - 2023-10-01T14:55:48.715 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-01T14:55:48.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "1;smithi031=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: Saving service mgr spec with placement smithi031=x;count:1 2023-10-01T14:55:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:55:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: Reconfiguring mgr.x (unknown last config time)... 2023-10-01T14:55:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T14:55:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T14:55:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:48.746 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:48 smithi031 ceph-mon[107152]: Reconfiguring daemon mgr.x on smithi031 2023-10-01T14:55:48.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "1;smithi031=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:48.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: Saving service mgr spec with placement smithi031=x;count:1 2023-10-01T14:55:48.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:48.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:48.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:48.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:48.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:55:48.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:48.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:48.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: Reconfiguring mgr.x (unknown last config time)... 2023-10-01T14:55:48.781 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T14:55:48.781 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T14:55:48.781 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:48.781 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:48 smithi099 ceph-mon[109132]: Reconfiguring daemon mgr.x on smithi031 2023-10-01T14:55:48.781 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2023-10-01T14:55:48.781 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2023-10-01T14:55:48.782 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000211728 s, 2.4 MB/s 2023-10-01T14:55:48.783 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-01T14:55:48.843 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_4 2023-10-01T14:55:48.901 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-01T14:55:48.901 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T14:55:48.901 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 6h/6d Inode: 388861 Links: 1 2023-10-01T14:55:48.901 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T14:55:48.901 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T14:55:48.901 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2023-10-01 14:55:36.599965093 +0000 2023-10-01T14:55:48.901 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2023-10-01 14:51:44.420360635 +0000 2023-10-01T14:55:48.901 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2023-10-01 14:51:44.420360635 +0000 2023-10-01T14:55:48.902 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: - 2023-10-01T14:55:48.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-01T14:55:48.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "1;smithi031=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:48.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: Saving service mgr spec with placement smithi031=x;count:1 2023-10-01T14:55:48.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:48.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:48.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:55:48.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:48.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:55:48.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:48.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:48.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: Reconfiguring mgr.x (unknown last config time)... 2023-10-01T14:55:48.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T14:55:48.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T14:55:48.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:48.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:48 smithi135 ceph-mon[109341]: Reconfiguring daemon mgr.x on smithi031 2023-10-01T14:55:48.967 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2023-10-01T14:55:48.967 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2023-10-01T14:55:48.967 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000238071 s, 2.2 MB/s 2023-10-01T14:55:48.968 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-01T14:55:49.027 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-10-01T14:55:49.027 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/scratch_devs of=/dev/stdout 2023-10-01T14:55:49.049 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-01T14:55:49.049 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_1 2023-10-01T14:55:49.110 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-01T14:55:49.111 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T14:55:49.111 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 6h/6d Inode: 390611 Links: 1 2023-10-01T14:55:49.111 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T14:55:49.111 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T14:55:49.111 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2023-10-01 14:55:29.335796101 +0000 2023-10-01T14:55:49.111 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2023-10-01 14:51:44.139538324 +0000 2023-10-01T14:55:49.111 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2023-10-01 14:51:44.139538324 +0000 2023-10-01T14:55:49.111 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: - 2023-10-01T14:55:49.111 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-01T14:55:49.183 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2023-10-01T14:55:49.184 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2023-10-01T14:55:49.184 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000217114 s, 2.4 MB/s 2023-10-01T14:55:49.185 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-01T14:55:49.246 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_2 2023-10-01T14:55:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-01T14:55:49.307 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T14:55:49.307 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 6h/6d Inode: 390716 Links: 1 2023-10-01T14:55:49.307 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T14:55:49.307 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T14:55:49.307 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2023-10-01 14:55:29.335796101 +0000 2023-10-01T14:55:49.308 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2023-10-01 14:51:44.623528516 +0000 2023-10-01T14:55:49.308 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2023-10-01 14:51:44.623528516 +0000 2023-10-01T14:55:49.308 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: - 2023-10-01T14:55:49.308 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-01T14:55:49.378 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2023-10-01T14:55:49.378 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2023-10-01T14:55:49.378 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000232633 s, 2.2 MB/s 2023-10-01T14:55:49.379 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-01T14:55:49.440 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_3 2023-10-01T14:55:49.500 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-01T14:55:49.500 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T14:55:49.500 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 6h/6d Inode: 390860 Links: 1 2023-10-01T14:55:49.501 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T14:55:49.501 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T14:55:49.501 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2023-10-01 14:55:29.335796101 +0000 2023-10-01T14:55:49.501 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2023-10-01 14:51:45.115518546 +0000 2023-10-01T14:55:49.501 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2023-10-01 14:51:45.115518546 +0000 2023-10-01T14:55:49.501 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: - 2023-10-01T14:55:49.501 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-01T14:55:49.569 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2023-10-01T14:55:49.569 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2023-10-01T14:55:49.570 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000219199 s, 2.3 MB/s 2023-10-01T14:55:49.570 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-01T14:55:49.631 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_4 2023-10-01T14:55:49.689 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-01T14:55:49.689 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T14:55:49.689 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 6h/6d Inode: 395160 Links: 1 2023-10-01T14:55:49.689 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T14:55:49.689 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T14:55:49.689 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2023-10-01 14:55:29.335796101 +0000 2023-10-01T14:55:49.689 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2023-10-01 14:51:45.614508433 +0000 2023-10-01T14:55:49.690 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2023-10-01 14:51:45.614508433 +0000 2023-10-01T14:55:49.690 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: - 2023-10-01T14:55:49.690 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-01T14:55:49.757 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2023-10-01T14:55:49.757 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2023-10-01T14:55:49.757 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000234947 s, 2.2 MB/s 2023-10-01T14:55:49.758 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-01T14:55:49.818 INFO:tasks.cephadm:Deploying osd.0 on smithi031 with /dev/vg_nvme/lv_4... 2023-10-01T14:55:49.818 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-01T14:55:49.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:49 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:49.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:49 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:49.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:49 smithi099 ceph-mon[109132]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:49.851 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:49 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:49.851 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:49 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:49.851 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:49 smithi031 ceph-mon[107152]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:49.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:49 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:49.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:49 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:55:49.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:49 smithi135 ceph-mon[109341]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:50.064 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:55:51.421 INFO:teuthology.orchestra.run.smithi031.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-01T14:55:51.422 INFO:teuthology.orchestra.run.smithi031.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-01T14:55:51.422 INFO:teuthology.orchestra.run.smithi031.stderr: stderr: 10+0 records in 2023-10-01T14:55:51.422 INFO:teuthology.orchestra.run.smithi031.stderr:10+0 records out 2023-10-01T14:55:51.422 INFO:teuthology.orchestra.run.smithi031.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0119458 s, 878 MB/s 2023-10-01T14:55:51.422 INFO:teuthology.orchestra.run.smithi031.stderr:--> Zapping successful for: 2023-10-01T14:55:51.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:51 smithi031 ceph-mon[107152]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:51.821 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi031:vg_nvme/lv_4 2023-10-01T14:55:52.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:51 smithi099 ceph-mon[109132]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:52.113 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:55:52.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:51 smithi135 ceph-mon[109341]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:53.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:53 smithi031 ceph-mon[107152]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:53.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:53 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:55:53.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:53 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:55:53.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:53 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:54.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:53 smithi099 ceph-mon[109132]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:54.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:53 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:55:54.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:53 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:55:54.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:53 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:54.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:53 smithi135 ceph-mon[109341]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:54.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:53 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:55:54.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:53 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:55:54.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:53 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:55:54.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:54 smithi031 ceph-mon[107152]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:55.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:54 smithi099 ceph-mon[109132]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:55.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:54 smithi135 ceph-mon[109341]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:55:55.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:55 smithi031 ceph-mon[107152]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:55.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:55 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/70899288' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d3a06bc-29dd-4824-847c-173f7a20a4d3"}]: dispatch 2023-10-01T14:55:55.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:55 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/70899288' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1d3a06bc-29dd-4824-847c-173f7a20a4d3"}]': finished 2023-10-01T14:55:55.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:55 smithi031 ceph-mon[107152]: osdmap e5: 1 total, 0 up, 1 in 2023-10-01T14:55:55.996 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:55 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:55:56.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:55 smithi099 ceph-mon[109132]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:56.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:55 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/70899288' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d3a06bc-29dd-4824-847c-173f7a20a4d3"}]: dispatch 2023-10-01T14:55:56.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:55 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/70899288' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1d3a06bc-29dd-4824-847c-173f7a20a4d3"}]': finished 2023-10-01T14:55:56.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:55 smithi099 ceph-mon[109132]: osdmap e5: 1 total, 0 up, 1 in 2023-10-01T14:55:56.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:55 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:55:56.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:55 smithi135 ceph-mon[109341]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:56.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:55 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/70899288' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d3a06bc-29dd-4824-847c-173f7a20a4d3"}]: dispatch 2023-10-01T14:55:56.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:55 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/70899288' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1d3a06bc-29dd-4824-847c-173f7a20a4d3"}]': finished 2023-10-01T14:55:56.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:55 smithi135 ceph-mon[109341]: osdmap e5: 1 total, 0 up, 1 in 2023-10-01T14:55:56.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:55 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:55:56.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:56 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/520761305' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:55:57.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:56 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/520761305' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:55:57.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:56 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/520761305' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:55:57.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:57 smithi031 ceph-mon[107152]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:58.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:57 smithi099 ceph-mon[109132]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:58.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:57 smithi135 ceph-mon[109341]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:55:59.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:55:59 smithi031 ceph-mon[107152]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:00.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:55:59 smithi099 ceph-mon[109132]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:00.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:55:59 smithi135 ceph-mon[109341]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:00.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T14:56:00.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:01.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T14:56:01.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:01.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T14:56:01.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:01.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:01 smithi031 ceph-mon[107152]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:01.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:01 smithi031 ceph-mon[107152]: Deploying daemon osd.0 on smithi031 2023-10-01T14:56:02.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:01 smithi099 ceph-mon[109132]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:02.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:01 smithi099 ceph-mon[109132]: Deploying daemon osd.0 on smithi031 2023-10-01T14:56:02.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:01 smithi135 ceph-mon[109341]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:02.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:01 smithi135 ceph-mon[109341]: Deploying daemon osd.0 on smithi031 2023-10-01T14:56:03.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:03 smithi031 ceph-mon[107152]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:04.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:03 smithi099 ceph-mon[109132]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:04.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:03 smithi135 ceph-mon[109341]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:05.862 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:05 smithi031 ceph-mon[107152]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:05.862 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:05 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:05.862 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:05 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:05.863 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:05 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:06.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:05 smithi099 ceph-mon[109132]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:06.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:05 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:06.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:05 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:06.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:05 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:06.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:05 smithi135 ceph-mon[109341]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:06.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:05 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:06.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:05 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:06.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:05 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:06.940 INFO:teuthology.orchestra.run.smithi031.stdout:Created osd(s) 0 on host 'smithi031' 2023-10-01T14:56:07.369 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.369 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.369 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:07 smithi031 ceph-mon[107152]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:07.369 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:07.369 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:07.369 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.370 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.370 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.370 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:07.371 DEBUG:teuthology.orchestra.run.smithi031:osd.0> sudo journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-01T14:56:07.373 INFO:tasks.cephadm:Deploying osd.1 on smithi031 with /dev/vg_nvme/lv_3... 2023-10-01T14:56:07.373 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-01T14:56:07.411 INFO:journalctl@ceph.osd.0.smithi031.stdout:-- Logs begin at Sun 2023-10-01 14:44:31 UTC. -- 2023-10-01T14:56:07.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:07 smithi099 ceph-mon[109132]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:07.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:07.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:07.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:07.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:07 smithi135 ceph-mon[109341]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:07.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:07.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:07.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:07.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:07.959 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:56:08.122 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 14:56:07 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-0[118577]: 2023-10-01T14:56:07.882+0000 7fce6669d640 -1 osd.0 0 log_to_monitors true 2023-10-01T14:56:08.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:08 smithi031 ceph-mon[107152]: from='osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T14:56:08.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:08 smithi099 ceph-mon[109132]: from='osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T14:56:08.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:08 smithi135 ceph-mon[109341]: from='osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T14:56:09.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:09 smithi099 ceph-mon[109132]: from='osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T14:56:09.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:09 smithi099 ceph-mon[109132]: osdmap e6: 1 total, 0 up, 1 in 2023-10-01T14:56:09.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:09 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:56:09.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:09 smithi099 ceph-mon[109132]: from='osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2023-10-01T14:56:09.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:09 smithi099 ceph-mon[109132]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:09.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:09 smithi135 ceph-mon[109341]: from='osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T14:56:09.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:09 smithi135 ceph-mon[109341]: osdmap e6: 1 total, 0 up, 1 in 2023-10-01T14:56:09.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:09 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:56:09.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:09 smithi135 ceph-mon[109341]: from='osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2023-10-01T14:56:09.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:09 smithi135 ceph-mon[109341]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:09.681 INFO:teuthology.orchestra.run.smithi031.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-01T14:56:09.681 INFO:teuthology.orchestra.run.smithi031.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-01T14:56:09.681 INFO:teuthology.orchestra.run.smithi031.stderr: stderr: 10+0 records in 2023-10-01T14:56:09.681 INFO:teuthology.orchestra.run.smithi031.stderr:10+0 records out 2023-10-01T14:56:09.681 INFO:teuthology.orchestra.run.smithi031.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0127898 s, 820 MB/s 2023-10-01T14:56:09.681 INFO:teuthology.orchestra.run.smithi031.stderr:--> Zapping successful for: 2023-10-01T14:56:09.721 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 14:56:09 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-0[118577]: 2023-10-01T14:56:09.282+0000 7fce5bd15700 -1 osd.0 0 waiting for initial osdmap 2023-10-01T14:56:09.721 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 14:56:09 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-0[118577]: 2023-10-01T14:56:09.284+0000 7fce54b04700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T14:56:09.722 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:09 smithi031 ceph-mon[107152]: from='osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T14:56:09.722 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:09 smithi031 ceph-mon[107152]: osdmap e6: 1 total, 0 up, 1 in 2023-10-01T14:56:09.722 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:09 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:56:09.722 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:09 smithi031 ceph-mon[107152]: from='osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2023-10-01T14:56:09.722 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:09 smithi031 ceph-mon[107152]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:10.120 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi031:vg_nvme/lv_3 2023-10-01T14:56:10.372 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:56:10.539 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:10 smithi031 ceph-mon[107152]: from='osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2023-10-01T14:56:10.539 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:10 smithi031 ceph-mon[107152]: osdmap e7: 1 total, 0 up, 1 in 2023-10-01T14:56:10.539 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:56:10.539 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:56:10.539 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:10 smithi031 ceph-mon[107152]: Detected new or changed devices on smithi031 2023-10-01T14:56:10.540 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:10.540 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:10.540 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:10.540 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:10 smithi031 ceph-mon[107152]: Adjusting osd_memory_target on smithi031 to 17118M 2023-10-01T14:56:10.540 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:10.540 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:10.540 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:10.540 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:10.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:10 smithi099 ceph-mon[109132]: from='osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2023-10-01T14:56:10.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:10 smithi099 ceph-mon[109132]: osdmap e7: 1 total, 0 up, 1 in 2023-10-01T14:56:10.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:56:10.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:56:10.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:10 smithi099 ceph-mon[109132]: Detected new or changed devices on smithi031 2023-10-01T14:56:10.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:10.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:10.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:10.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:10 smithi099 ceph-mon[109132]: Adjusting osd_memory_target on smithi031 to 17118M 2023-10-01T14:56:10.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:10.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:10.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:10.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:10.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:10 smithi135 ceph-mon[109341]: from='osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2023-10-01T14:56:10.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:10 smithi135 ceph-mon[109341]: osdmap e7: 1 total, 0 up, 1 in 2023-10-01T14:56:10.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:56:10.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:56:10.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:10 smithi135 ceph-mon[109341]: Detected new or changed devices on smithi031 2023-10-01T14:56:10.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:10.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:10.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:10.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:10 smithi135 ceph-mon[109341]: Adjusting osd_memory_target on smithi031 to 17118M 2023-10-01T14:56:10.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:10.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:10.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:10.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:11.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:11 smithi099 ceph-mon[109132]: purged_snaps scrub starts 2023-10-01T14:56:11.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:11 smithi099 ceph-mon[109132]: purged_snaps scrub ok 2023-10-01T14:56:11.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:11 smithi099 ceph-mon[109132]: osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266] boot 2023-10-01T14:56:11.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:11 smithi099 ceph-mon[109132]: osdmap e8: 1 total, 1 up, 1 in 2023-10-01T14:56:11.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:11 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:56:11.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:11 smithi099 ceph-mon[109132]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:11.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:11 smithi135 ceph-mon[109341]: purged_snaps scrub starts 2023-10-01T14:56:11.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:11 smithi135 ceph-mon[109341]: purged_snaps scrub ok 2023-10-01T14:56:11.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:11 smithi135 ceph-mon[109341]: osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266] boot 2023-10-01T14:56:11.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:11 smithi135 ceph-mon[109341]: osdmap e8: 1 total, 1 up, 1 in 2023-10-01T14:56:11.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:11 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:56:11.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:11 smithi135 ceph-mon[109341]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:11.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:11 smithi031 ceph-mon[107152]: purged_snaps scrub starts 2023-10-01T14:56:11.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:11 smithi031 ceph-mon[107152]: purged_snaps scrub ok 2023-10-01T14:56:11.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:11 smithi031 ceph-mon[107152]: osd.0 [v2:172.21.15.31:6802/3666492266,v1:172.21.15.31:6803/3666492266] boot 2023-10-01T14:56:11.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:11 smithi031 ceph-mon[107152]: osdmap e8: 1 total, 1 up, 1 in 2023-10-01T14:56:11.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:11 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T14:56:11.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:11 smithi031 ceph-mon[107152]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T14:56:12.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:12 smithi099 ceph-mon[109132]: osdmap e9: 1 total, 1 up, 1 in 2023-10-01T14:56:12.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:12 smithi099 ceph-mon[109132]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:56:12.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:12 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:56:12.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:12 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:56:12.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:12 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:12.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:12 smithi135 ceph-mon[109341]: osdmap e9: 1 total, 1 up, 1 in 2023-10-01T14:56:12.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:12 smithi135 ceph-mon[109341]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:56:12.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:12 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:56:12.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:12 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:56:12.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:12 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:12.698 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:12 smithi031 ceph-mon[107152]: osdmap e9: 1 total, 1 up, 1 in 2023-10-01T14:56:12.698 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:12 smithi031 ceph-mon[107152]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:56:12.698 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:56:12.698 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:56:12.698 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:13.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:13 smithi099 ceph-mon[109132]: pgmap v36: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:13.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:13 smithi135 ceph-mon[109341]: pgmap v36: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:13.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:13 smithi031 ceph-mon[107152]: pgmap v36: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:14.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:14 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/634024959' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "db968f7c-d7ed-447b-b74b-bf21a6ba0b4a"}]: dispatch 2023-10-01T14:56:14.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:14 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/634024959' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "db968f7c-d7ed-447b-b74b-bf21a6ba0b4a"}]': finished 2023-10-01T14:56:14.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:14 smithi099 ceph-mon[109132]: osdmap e10: 2 total, 1 up, 2 in 2023-10-01T14:56:14.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:14 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:14.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:14 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/2772353325' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:56:14.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:14 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/634024959' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "db968f7c-d7ed-447b-b74b-bf21a6ba0b4a"}]: dispatch 2023-10-01T14:56:14.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:14 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/634024959' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "db968f7c-d7ed-447b-b74b-bf21a6ba0b4a"}]': finished 2023-10-01T14:56:14.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:14 smithi135 ceph-mon[109341]: osdmap e10: 2 total, 1 up, 2 in 2023-10-01T14:56:14.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:14 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:14.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:14 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/2772353325' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:56:14.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:14 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/634024959' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "db968f7c-d7ed-447b-b74b-bf21a6ba0b4a"}]: dispatch 2023-10-01T14:56:14.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:14 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/634024959' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "db968f7c-d7ed-447b-b74b-bf21a6ba0b4a"}]': finished 2023-10-01T14:56:14.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:14 smithi031 ceph-mon[107152]: osdmap e10: 2 total, 1 up, 2 in 2023-10-01T14:56:14.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:14 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:14.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:14 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/2772353325' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:56:15.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:15 smithi099 ceph-mon[109132]: pgmap v38: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:15.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:15 smithi135 ceph-mon[109341]: pgmap v38: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:15.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:15 smithi031 ceph-mon[107152]: pgmap v38: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:17.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:17 smithi031 ceph-mon[107152]: pgmap v39: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:17.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:17 smithi099 ceph-mon[109132]: pgmap v39: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:17.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:17 smithi135 ceph-mon[109341]: pgmap v39: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:19.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:19 smithi031 ceph-mon[107152]: pgmap v40: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:19.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:19 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T14:56:19.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:19 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:19.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:19 smithi031 ceph-mon[107152]: Deploying daemon osd.1 on smithi031 2023-10-01T14:56:20.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:19 smithi099 ceph-mon[109132]: pgmap v40: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:20.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:19 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T14:56:20.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:19 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:20.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:19 smithi099 ceph-mon[109132]: Deploying daemon osd.1 on smithi031 2023-10-01T14:56:20.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:19 smithi135 ceph-mon[109341]: pgmap v40: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:20.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:19 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T14:56:20.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:19 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:20.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:19 smithi135 ceph-mon[109341]: Deploying daemon osd.1 on smithi031 2023-10-01T14:56:21.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:21 smithi031 ceph-mon[107152]: pgmap v41: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:22.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:21 smithi099 ceph-mon[109132]: pgmap v41: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:22.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:21 smithi135 ceph-mon[109341]: pgmap v41: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:23.929 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:23 smithi031 ceph-mon[107152]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:23.929 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:23 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:23.929 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:23 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:23.929 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:23 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:24.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:23 smithi099 ceph-mon[109132]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:24.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:23 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:24.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:23 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:24.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:23 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:24.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:23 smithi135 ceph-mon[109341]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:24.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:23 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:24.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:23 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:24.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:23 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:25.089 INFO:teuthology.orchestra.run.smithi031.stdout:Created osd(s) 1 on host 'smithi031' 2023-10-01T14:56:25.510 DEBUG:teuthology.orchestra.run.smithi031:osd.1> sudo journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-01T14:56:25.512 INFO:tasks.cephadm:Deploying osd.2 on smithi099 with /dev/vg_nvme/lv_4... 2023-10-01T14:56:25.512 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-01T14:56:25.541 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.541 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.542 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:25.542 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:25.542 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.542 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:25 smithi031 ceph-mon[107152]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:25.542 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.542 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.542 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:25 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:25.548 INFO:journalctl@ceph.osd.1.smithi031.stdout:-- Logs begin at Sun 2023-10-01 14:44:31 UTC. -- 2023-10-01T14:56:25.736 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T14:56:25.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:25 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:25 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:25 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:25.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:25 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:25.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:25 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:25 smithi099 ceph-mon[109132]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:25.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:25 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:25 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:25 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:25.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:25 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:25 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:25 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:25.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:25 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:25.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:25 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:25 smithi135 ceph-mon[109341]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:25.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:25 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:25 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:25.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:25 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:26.245 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 14:56:25 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-1[122845]: 2023-10-01T14:56:25.966+0000 7f589a9e3640 -1 osd.1 0 log_to_monitors true 2023-10-01T14:56:26.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:26 smithi031 ceph-mon[107152]: from='osd.1 [v2:172.21.15.31:6810/3064465530,v1:172.21.15.31:6811/3064465530]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T14:56:26.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:26 smithi031 ceph-mon[107152]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T14:56:26.805 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:26 smithi099 ceph-mon[109132]: from='osd.1 [v2:172.21.15.31:6810/3064465530,v1:172.21.15.31:6811/3064465530]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T14:56:26.805 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:26 smithi099 ceph-mon[109132]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T14:56:26.828 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-01T14:56:26.828 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-01T14:56:26.828 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2023-10-01T14:56:26.828 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2023-10-01T14:56:26.828 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111598 s, 940 MB/s 2023-10-01T14:56:26.828 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2023-10-01T14:56:26.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:26 smithi135 ceph-mon[109341]: from='osd.1 [v2:172.21.15.31:6810/3064465530,v1:172.21.15.31:6811/3064465530]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T14:56:26.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:26 smithi135 ceph-mon[109341]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T14:56:27.196 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi099:vg_nvme/lv_4 2023-10-01T14:56:27.410 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T14:56:27.482 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T14:56:27.482 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: osdmap e11: 2 total, 1 up, 2 in 2023-10-01T14:56:27.482 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:27.483 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: from='osd.1 [v2:172.21.15.31:6810/3064465530,v1:172.21.15.31:6811/3064465530]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2023-10-01T14:56:27.483 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2023-10-01T14:56:27.483 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:27.483 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:27.483 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:27.483 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:27.483 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:27.483 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:27.483 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:27.483 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:27.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:27.745 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-1[122845]: 2023-10-01T14:56:27.483+0000 7f589085c700 -1 osd.1 0 waiting for initial osdmap 2023-10-01T14:56:27.745 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 14:56:27 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-1[122845]: 2023-10-01T14:56:27.487+0000 7f5889e4c700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T14:56:27.802 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T14:56:27.802 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: osdmap e11: 2 total, 1 up, 2 in 2023-10-01T14:56:27.802 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:27.802 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: from='osd.1 [v2:172.21.15.31:6810/3064465530,v1:172.21.15.31:6811/3064465530]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2023-10-01T14:56:27.803 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2023-10-01T14:56:27.803 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:27.803 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:27.803 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:27.803 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:27.803 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:27.803 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:27.803 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:27.803 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:27.803 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:27 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:27.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T14:56:27.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: osdmap e11: 2 total, 1 up, 2 in 2023-10-01T14:56:27.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:27.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: from='osd.1 [v2:172.21.15.31:6810/3064465530,v1:172.21.15.31:6811/3064465530]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2023-10-01T14:56:27.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2023-10-01T14:56:27.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:27.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:27.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:27.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:27.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:27.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:27.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:27.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:27.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:27 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:28.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:28 smithi031 ceph-mon[107152]: Detected new or changed devices on smithi031 2023-10-01T14:56:28.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:28 smithi031 ceph-mon[107152]: Adjusting osd_memory_target on smithi031 to 8559M 2023-10-01T14:56:28.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:28 smithi031 ceph-mon[107152]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2023-10-01T14:56:28.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:28 smithi031 ceph-mon[107152]: osdmap e12: 2 total, 1 up, 2 in 2023-10-01T14:56:28.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:28 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:28.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:28 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:28.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:28 smithi099 ceph-mon[109132]: Detected new or changed devices on smithi031 2023-10-01T14:56:28.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:28 smithi099 ceph-mon[109132]: Adjusting osd_memory_target on smithi031 to 8559M 2023-10-01T14:56:28.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:28 smithi099 ceph-mon[109132]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2023-10-01T14:56:28.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:28 smithi099 ceph-mon[109132]: osdmap e12: 2 total, 1 up, 2 in 2023-10-01T14:56:28.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:28 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:28.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:28 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:28.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:28 smithi135 ceph-mon[109341]: Detected new or changed devices on smithi031 2023-10-01T14:56:28.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:28 smithi135 ceph-mon[109341]: Adjusting osd_memory_target on smithi031 to 8559M 2023-10-01T14:56:28.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:28 smithi135 ceph-mon[109341]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2023-10-01T14:56:28.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:28 smithi135 ceph-mon[109341]: osdmap e12: 2 total, 1 up, 2 in 2023-10-01T14:56:28.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:28 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:28.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:28 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:29.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:29 smithi099 ceph-mon[109132]: purged_snaps scrub starts 2023-10-01T14:56:29.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:29 smithi099 ceph-mon[109132]: purged_snaps scrub ok 2023-10-01T14:56:29.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:29 smithi099 ceph-mon[109132]: osd.1 [v2:172.21.15.31:6810/3064465530,v1:172.21.15.31:6811/3064465530] boot 2023-10-01T14:56:29.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:29 smithi099 ceph-mon[109132]: osdmap e13: 2 total, 2 up, 2 in 2023-10-01T14:56:29.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:29 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:29.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:29 smithi099 ceph-mon[109132]: from='client.24152 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:56:29.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:29 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:56:29.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:29 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:56:29.547 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:29 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:29.547 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:29 smithi099 ceph-mon[109132]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:29.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:29 smithi031 ceph-mon[107152]: purged_snaps scrub starts 2023-10-01T14:56:29.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:29 smithi031 ceph-mon[107152]: purged_snaps scrub ok 2023-10-01T14:56:29.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:29 smithi031 ceph-mon[107152]: osd.1 [v2:172.21.15.31:6810/3064465530,v1:172.21.15.31:6811/3064465530] boot 2023-10-01T14:56:29.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:29 smithi031 ceph-mon[107152]: osdmap e13: 2 total, 2 up, 2 in 2023-10-01T14:56:29.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:29 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:29.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:29 smithi031 ceph-mon[107152]: from='client.24152 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:56:29.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:29 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:56:29.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:29 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:56:29.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:29 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:29.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:29 smithi031 ceph-mon[107152]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:29.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:29 smithi135 ceph-mon[109341]: purged_snaps scrub starts 2023-10-01T14:56:29.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:29 smithi135 ceph-mon[109341]: purged_snaps scrub ok 2023-10-01T14:56:29.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:29 smithi135 ceph-mon[109341]: osd.1 [v2:172.21.15.31:6810/3064465530,v1:172.21.15.31:6811/3064465530] boot 2023-10-01T14:56:29.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:29 smithi135 ceph-mon[109341]: osdmap e13: 2 total, 2 up, 2 in 2023-10-01T14:56:29.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T14:56:29.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:29 smithi135 ceph-mon[109341]: from='client.24152 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:56:29.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:56:29.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:56:29.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:29 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:29.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:29 smithi135 ceph-mon[109341]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-01T14:56:30.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:30 smithi031 ceph-mon[107152]: osdmap e14: 2 total, 2 up, 2 in 2023-10-01T14:56:30.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:30 smithi031 ceph-mon[107152]: from='client.? 172.21.15.99:0/327988418' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7eb06643-5689-4ee7-94d9-58a39433c2ce"}]: dispatch 2023-10-01T14:56:30.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:30 smithi031 ceph-mon[107152]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7eb06643-5689-4ee7-94d9-58a39433c2ce"}]: dispatch 2023-10-01T14:56:30.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:30 smithi031 ceph-mon[107152]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7eb06643-5689-4ee7-94d9-58a39433c2ce"}]': finished 2023-10-01T14:56:30.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:30 smithi031 ceph-mon[107152]: osdmap e15: 3 total, 2 up, 3 in 2023-10-01T14:56:30.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:30 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:30.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:30 smithi099 ceph-mon[109132]: osdmap e14: 2 total, 2 up, 2 in 2023-10-01T14:56:30.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:30 smithi099 ceph-mon[109132]: from='client.? 172.21.15.99:0/327988418' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7eb06643-5689-4ee7-94d9-58a39433c2ce"}]: dispatch 2023-10-01T14:56:30.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:30 smithi099 ceph-mon[109132]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7eb06643-5689-4ee7-94d9-58a39433c2ce"}]: dispatch 2023-10-01T14:56:30.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:30 smithi099 ceph-mon[109132]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7eb06643-5689-4ee7-94d9-58a39433c2ce"}]': finished 2023-10-01T14:56:30.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:30 smithi099 ceph-mon[109132]: osdmap e15: 3 total, 2 up, 3 in 2023-10-01T14:56:30.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:30 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:30.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:30 smithi135 ceph-mon[109341]: osdmap e14: 2 total, 2 up, 2 in 2023-10-01T14:56:30.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:30 smithi135 ceph-mon[109341]: from='client.? 172.21.15.99:0/327988418' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7eb06643-5689-4ee7-94d9-58a39433c2ce"}]: dispatch 2023-10-01T14:56:30.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:30 smithi135 ceph-mon[109341]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7eb06643-5689-4ee7-94d9-58a39433c2ce"}]: dispatch 2023-10-01T14:56:30.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:30 smithi135 ceph-mon[109341]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7eb06643-5689-4ee7-94d9-58a39433c2ce"}]': finished 2023-10-01T14:56:30.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:30 smithi135 ceph-mon[109341]: osdmap e15: 3 total, 2 up, 3 in 2023-10-01T14:56:30.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:30 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:31.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:31 smithi099 ceph-mon[109132]: pgmap v51: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:31.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:31 smithi099 ceph-mon[109132]: from='client.? 172.21.15.99:0/393343895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:56:31.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:31 smithi135 ceph-mon[109341]: pgmap v51: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:31.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:31 smithi135 ceph-mon[109341]: from='client.? 172.21.15.99:0/393343895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:56:31.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:31 smithi031 ceph-mon[107152]: pgmap v51: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:31.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:31 smithi031 ceph-mon[107152]: from='client.? 172.21.15.99:0/393343895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:56:33.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:33 smithi031 ceph-mon[107152]: pgmap v52: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:34.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:33 smithi099 ceph-mon[109132]: pgmap v52: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:34.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:33 smithi135 ceph-mon[109341]: pgmap v52: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:35.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:35 smithi099 ceph-mon[109132]: pgmap v53: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:35.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:35 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T14:56:35.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:35 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:35.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:35 smithi031 ceph-mon[107152]: pgmap v53: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:35.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:35 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T14:56:35.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:35 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:36.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:35 smithi135 ceph-mon[109341]: pgmap v53: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:36.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:35 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T14:56:36.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:35 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:36.967 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:36 smithi099 ceph-mon[109132]: Deploying daemon osd.2 on smithi099 2023-10-01T14:56:36.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:36 smithi031 ceph-mon[107152]: Deploying daemon osd.2 on smithi099 2023-10-01T14:56:37.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:36 smithi135 ceph-mon[109341]: Deploying daemon osd.2 on smithi099 2023-10-01T14:56:37.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:37 smithi031 ceph-mon[107152]: pgmap v54: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:38.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:37 smithi099 ceph-mon[109132]: pgmap v54: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:38.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:37 smithi135 ceph-mon[109341]: pgmap v54: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:39.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:39 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:39.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:39 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:39.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:39 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:39.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:39 smithi099 ceph-mon[109132]: pgmap v55: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:39.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:39 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:39.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:39 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:39.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:39 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:39.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:39 smithi135 ceph-mon[109341]: pgmap v55: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:39.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:39 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:39.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:39 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:39.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:39 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:39.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:39 smithi031 ceph-mon[107152]: pgmap v55: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:40.423 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 2 on host 'smithi099' 2023-10-01T14:56:40.804 DEBUG:teuthology.orchestra.run.smithi099:osd.2> sudo journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-01T14:56:40.805 INFO:tasks.cephadm:Deploying osd.3 on smithi099 with /dev/vg_nvme/lv_3... 2023-10-01T14:56:40.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-01T14:56:40.836 INFO:journalctl@ceph.osd.2.smithi099.stdout:-- Logs begin at Sun 2023-10-01 14:44:43 UTC. -- 2023-10-01T14:56:41.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:40 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:40 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:40 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:41.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:40 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:41.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:40 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:40 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:40 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:40 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:41.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:40 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:40 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:40 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:41.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:40 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:41.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:40 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:40 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:40 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:40 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:41.367 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T14:56:41.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:40 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:40 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:40 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:41.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:40 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:41.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:40 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:40 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:40 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:41.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:40 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:42.089 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 14:56:41 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-2[113053]: 2023-10-01T14:56:41.744+0000 7ff5ee39a640 -1 osd.2 0 log_to_monitors true 2023-10-01T14:56:42.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:41 smithi099 ceph-mon[109132]: pgmap v56: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:42.090 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:41 smithi099 ceph-mon[109132]: from='osd.2 [v2:172.21.15.99:6800/563131629,v1:172.21.15.99:6801/563131629]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T14:56:42.090 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:41 smithi099 ceph-mon[109132]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T14:56:42.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:41 smithi031 ceph-mon[107152]: pgmap v56: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:42.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:41 smithi031 ceph-mon[107152]: from='osd.2 [v2:172.21.15.99:6800/563131629,v1:172.21.15.99:6801/563131629]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T14:56:42.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:41 smithi031 ceph-mon[107152]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T14:56:42.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:41 smithi135 ceph-mon[109341]: pgmap v56: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:42.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:41 smithi135 ceph-mon[109341]: from='osd.2 [v2:172.21.15.99:6800/563131629,v1:172.21.15.99:6801/563131629]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T14:56:42.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:41 smithi135 ceph-mon[109341]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T14:56:42.645 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-01T14:56:42.646 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-01T14:56:42.646 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2023-10-01T14:56:42.646 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2023-10-01T14:56:42.646 INFO:teuthology.orchestra.run.smithi099.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.011335 s, 925 MB/s 2023-10-01T14:56:42.646 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 2023-10-01T14:56:42.646 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2023-10-01T14:56:43.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi099:vg_nvme/lv_3 2023-10-01T14:56:43.364 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T14:56:43.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T14:56:43.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: osdmap e16: 3 total, 2 up, 3 in 2023-10-01T14:56:43.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:43.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: from='osd.2 [v2:172.21.15.99:6800/563131629,v1:172.21.15.99:6801/563131629]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:56:43.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:56:43.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: pgmap v58: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:43.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: Detected new or changed devices on smithi099 2023-10-01T14:56:43.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:43.673 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:43.673 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:43.673 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: Adjusting osd_memory_target on smithi099 to 21209M 2023-10-01T14:56:43.673 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:43.673 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:43.673 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:43.673 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:43.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T14:56:43.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: osdmap e16: 3 total, 2 up, 3 in 2023-10-01T14:56:43.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: from='osd.2 [v2:172.21.15.99:6800/563131629,v1:172.21.15.99:6801/563131629]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:56:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:56:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: pgmap v58: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: Detected new or changed devices on smithi099 2023-10-01T14:56:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: Adjusting osd_memory_target on smithi099 to 21209M 2023-10-01T14:56:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:43.825 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-2[113053]: 2023-10-01T14:56:43.397+0000 7ff5e3a12700 -1 osd.2 0 waiting for initial osdmap 2023-10-01T14:56:43.825 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-2[113053]: 2023-10-01T14:56:43.400+0000 7ff5dd803700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T14:56:43.825 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T14:56:43.825 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: osdmap e16: 3 total, 2 up, 3 in 2023-10-01T14:56:43.825 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:43.825 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: from='osd.2 [v2:172.21.15.99:6800/563131629,v1:172.21.15.99:6801/563131629]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:56:43.825 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:56:43.825 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: pgmap v58: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:43.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: Detected new or changed devices on smithi099 2023-10-01T14:56:43.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:43.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:43.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:43.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: Adjusting osd_memory_target on smithi099 to 21209M 2023-10-01T14:56:43.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:43.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:43.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:43.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:44 smithi135 ceph-mon[109341]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-10-01T14:56:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:44 smithi135 ceph-mon[109341]: osdmap e17: 3 total, 2 up, 3 in 2023-10-01T14:56:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:44.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:44.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:44 smithi031 ceph-mon[107152]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-10-01T14:56:44.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:44 smithi031 ceph-mon[107152]: osdmap e17: 3 total, 2 up, 3 in 2023-10-01T14:56:44.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:44.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:44.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:44.809 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:44 smithi099 ceph-mon[109132]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-10-01T14:56:44.809 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:44 smithi099 ceph-mon[109132]: osdmap e17: 3 total, 2 up, 3 in 2023-10-01T14:56:44.809 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:44.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:44.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:45.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:45 smithi135 ceph-mon[109341]: purged_snaps scrub starts 2023-10-01T14:56:45.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:45 smithi135 ceph-mon[109341]: purged_snaps scrub ok 2023-10-01T14:56:45.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:45 smithi135 ceph-mon[109341]: osd.2 [v2:172.21.15.99:6800/563131629,v1:172.21.15.99:6801/563131629] boot 2023-10-01T14:56:45.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:45 smithi135 ceph-mon[109341]: osdmap e18: 3 total, 3 up, 3 in 2023-10-01T14:56:45.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:45 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:45.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:45 smithi135 ceph-mon[109341]: from='client.24179 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:56:45.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:45 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:56:45.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:45 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:56:45.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:45 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:45.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:45 smithi135 ceph-mon[109341]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:45.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:45 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2023-10-01T14:56:45.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:45 smithi031 ceph-mon[107152]: purged_snaps scrub starts 2023-10-01T14:56:45.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:45 smithi031 ceph-mon[107152]: purged_snaps scrub ok 2023-10-01T14:56:45.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:45 smithi031 ceph-mon[107152]: osd.2 [v2:172.21.15.99:6800/563131629,v1:172.21.15.99:6801/563131629] boot 2023-10-01T14:56:45.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:45 smithi031 ceph-mon[107152]: osdmap e18: 3 total, 3 up, 3 in 2023-10-01T14:56:45.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:45 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:45.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:45 smithi031 ceph-mon[107152]: from='client.24179 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:56:45.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:45 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:56:45.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:45 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:56:45.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:45 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:45.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:45 smithi031 ceph-mon[107152]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:45.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:45 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2023-10-01T14:56:45.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:45 smithi099 ceph-mon[109132]: purged_snaps scrub starts 2023-10-01T14:56:45.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:45 smithi099 ceph-mon[109132]: purged_snaps scrub ok 2023-10-01T14:56:45.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:45 smithi099 ceph-mon[109132]: osd.2 [v2:172.21.15.99:6800/563131629,v1:172.21.15.99:6801/563131629] boot 2023-10-01T14:56:45.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:45 smithi099 ceph-mon[109132]: osdmap e18: 3 total, 3 up, 3 in 2023-10-01T14:56:45.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:45 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T14:56:45.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:45 smithi099 ceph-mon[109132]: from='client.24179 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:56:45.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:45 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:56:45.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:45 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:56:45.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:45 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:45.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:45 smithi099 ceph-mon[109132]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-01T14:56:45.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:45 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2023-10-01T14:56:46.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:46 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2023-10-01T14:56:46.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:46 smithi135 ceph-mon[109341]: osdmap e19: 3 total, 3 up, 3 in 2023-10-01T14:56:46.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:46 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T14:56:46.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:46 smithi135 ceph-mon[109341]: from='client.? 172.21.15.99:0/2111343031' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a17c4e0-6068-4031-b08a-1bec022f3752"}]: dispatch 2023-10-01T14:56:46.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:46 smithi135 ceph-mon[109341]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a17c4e0-6068-4031-b08a-1bec022f3752"}]: dispatch 2023-10-01T14:56:46.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:46 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T14:56:46.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:46 smithi135 ceph-mon[109341]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0a17c4e0-6068-4031-b08a-1bec022f3752"}]': finished 2023-10-01T14:56:46.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:46 smithi135 ceph-mon[109341]: osdmap e20: 4 total, 3 up, 4 in 2023-10-01T14:56:46.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:46 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:56:46.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:46 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2023-10-01T14:56:46.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:46 smithi031 ceph-mon[107152]: osdmap e19: 3 total, 3 up, 3 in 2023-10-01T14:56:46.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:46 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T14:56:46.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:46 smithi031 ceph-mon[107152]: from='client.? 172.21.15.99:0/2111343031' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a17c4e0-6068-4031-b08a-1bec022f3752"}]: dispatch 2023-10-01T14:56:46.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:46 smithi031 ceph-mon[107152]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a17c4e0-6068-4031-b08a-1bec022f3752"}]: dispatch 2023-10-01T14:56:46.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:46 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T14:56:46.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:46 smithi031 ceph-mon[107152]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0a17c4e0-6068-4031-b08a-1bec022f3752"}]': finished 2023-10-01T14:56:46.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:46 smithi031 ceph-mon[107152]: osdmap e20: 4 total, 3 up, 4 in 2023-10-01T14:56:46.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:46 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:56:46.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:46 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2023-10-01T14:56:46.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:46 smithi099 ceph-mon[109132]: osdmap e19: 3 total, 3 up, 3 in 2023-10-01T14:56:46.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:46 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T14:56:46.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:46 smithi099 ceph-mon[109132]: from='client.? 172.21.15.99:0/2111343031' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a17c4e0-6068-4031-b08a-1bec022f3752"}]: dispatch 2023-10-01T14:56:46.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:46 smithi099 ceph-mon[109132]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a17c4e0-6068-4031-b08a-1bec022f3752"}]: dispatch 2023-10-01T14:56:46.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:46 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T14:56:46.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:46 smithi099 ceph-mon[109132]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0a17c4e0-6068-4031-b08a-1bec022f3752"}]': finished 2023-10-01T14:56:46.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:46 smithi099 ceph-mon[109132]: osdmap e20: 4 total, 3 up, 4 in 2023-10-01T14:56:46.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:46 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:56:47.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:47 smithi031 ceph-mon[107152]: pgmap v64: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:47.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:47 smithi031 ceph-mon[107152]: from='client.? 172.21.15.99:0/2375550200' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:56:47.745 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 14:56:47 smithi031 sudo[124717]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-01T14:56:47.745 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 14:56:47 smithi031 sudo[124717]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-01T14:56:47.745 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 14:56:47 smithi031 sudo[124717]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-01T14:56:47.745 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 14:56:47 smithi031 sudo[124717]: pam_unix(sudo:session): session closed for user root 2023-10-01T14:56:47.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:47 smithi099 ceph-mon[109132]: pgmap v64: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:47.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:47 smithi099 ceph-mon[109132]: from='client.? 172.21.15.99:0/2375550200' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:56:47.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:47 smithi135 ceph-mon[109341]: pgmap v64: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:47.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:47 smithi135 ceph-mon[109341]: from='client.? 172.21.15.99:0/2375550200' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:56:48.244 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 14:56:47 smithi031 sudo[124728]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-01T14:56:48.244 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 14:56:47 smithi031 sudo[124728]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-01T14:56:48.244 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 14:56:47 smithi031 sudo[124728]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-01T14:56:48.244 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 14:56:47 smithi031 sudo[124728]: pam_unix(sudo:session): session closed for user root 2023-10-01T14:56:48.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:48 smithi031 ceph-mon[107152]: mgrmap e14: x(active, since 108s) 2023-10-01T14:56:48.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:48 smithi031 ceph-mon[107152]: osdmap e21: 4 total, 3 up, 4 in 2023-10-01T14:56:48.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:56:48.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:48.745 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 14:56:48 smithi031 sudo[124731]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-01T14:56:48.745 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 14:56:48 smithi031 sudo[124731]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-01T14:56:48.745 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 14:56:48 smithi031 sudo[124731]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-01T14:56:48.745 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 14:56:48 smithi031 sudo[124731]: pam_unix(sudo:session): session closed for user root 2023-10-01T14:56:48.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:48 smithi099 ceph-mon[109132]: mgrmap e14: x(active, since 108s) 2023-10-01T14:56:48.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:48 smithi099 ceph-mon[109132]: osdmap e21: 4 total, 3 up, 4 in 2023-10-01T14:56:48.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:56:48.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:48.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:48 smithi135 ceph-mon[109341]: mgrmap e14: x(active, since 108s) 2023-10-01T14:56:48.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:48 smithi135 ceph-mon[109341]: osdmap e21: 4 total, 3 up, 4 in 2023-10-01T14:56:48.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:56:48.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:49.241 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 14:56:49 smithi099 sudo[116331]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-01T14:56:49.241 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 14:56:49 smithi099 sudo[116331]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-01T14:56:49.241 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 14:56:49 smithi099 sudo[116331]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-01T14:56:49.241 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 14:56:49 smithi099 sudo[116331]: pam_unix(sudo:session): session closed for user root 2023-10-01T14:56:49.244 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 14:56:48 smithi031 sudo[124734]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-01T14:56:49.245 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 14:56:48 smithi031 sudo[124734]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-01T14:56:49.245 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 14:56:48 smithi031 sudo[124734]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-01T14:56:49.245 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 14:56:48 smithi031 sudo[124734]: pam_unix(sudo:session): session closed for user root 2023-10-01T14:56:49.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:49 smithi099 ceph-mon[109132]: pgmap v66: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:49.589 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 14:56:49 smithi099 sudo[116409]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-01T14:56:49.589 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 14:56:49 smithi099 sudo[116409]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-01T14:56:49.589 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 14:56:49 smithi099 sudo[116409]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-01T14:56:49.589 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 14:56:49 smithi099 sudo[116409]: pam_unix(sudo:session): session closed for user root 2023-10-01T14:56:49.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:49 smithi031 ceph-mon[107152]: pgmap v66: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:49.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:49 smithi135 ceph-mon[109341]: pgmap v66: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:50.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:49 smithi031 sudo[124737]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-01T14:56:50.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:49 smithi031 sudo[124737]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-01T14:56:50.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:49 smithi031 sudo[124737]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-01T14:56:50.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:49 smithi031 sudo[124737]: pam_unix(sudo:session): session closed for user root 2023-10-01T14:56:50.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:50 smithi031 sudo[124740]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-01T14:56:50.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:50 smithi031 sudo[124740]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-01T14:56:50.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:50 smithi031 sudo[124740]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-01T14:56:50.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:50 smithi031 sudo[124740]: pam_unix(sudo:session): session closed for user root 2023-10-01T14:56:50.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 sudo[111478]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-01T14:56:50.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 sudo[111478]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-01T14:56:50.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 sudo[111478]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-01T14:56:50.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 sudo[111478]: pam_unix(sudo:session): session closed for user root 2023-10-01T14:56:50.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 sudo[111481]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-01T14:56:50.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 sudo[111481]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-01T14:56:50.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 sudo[111481]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-01T14:56:50.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 sudo[111481]: pam_unix(sudo:session): session closed for user root 2023-10-01T14:56:50.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:50.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 ceph-mon[109341]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T14:56:50.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 ceph-mon[109341]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T14:56:50.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:56:50.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:56:50.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:56:50.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:56:50.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:56:50.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:50 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:56:51.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:50 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:51.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:50 smithi099 ceph-mon[109132]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T14:56:51.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:50 smithi099 ceph-mon[109132]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T14:56:51.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:50 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:56:51.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:50 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:56:51.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:50 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:56:51.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:50 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:56:51.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:50 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:56:51.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:50 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:56:51.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:50 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:51.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:50 smithi031 ceph-mon[107152]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T14:56:51.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:50 smithi031 ceph-mon[107152]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T14:56:51.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:50 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:56:51.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:50 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:56:51.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:50 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:56:51.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:50 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:56:51.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:50 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:56:51.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:50 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:56:51.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 sudo[116630]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-01T14:56:51.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 sudo[116630]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-01T14:56:51.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 sudo[116630]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-01T14:56:51.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 sudo[116630]: pam_unix(sudo:session): session closed for user root 2023-10-01T14:56:51.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 sudo[116680]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-01T14:56:51.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 sudo[116680]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-01T14:56:51.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 sudo[116680]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-01T14:56:51.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 sudo[116680]: pam_unix(sudo:session): session closed for user root 2023-10-01T14:56:51.996 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 ceph-mon[109132]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T14:56:51.997 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 ceph-mon[109132]: pgmap v67: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:51.997 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 ceph-mon[109132]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T14:56:51.997 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:51.997 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 ceph-mon[109132]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T14:56:51.997 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:56:51.997 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:56:51.997 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:56:51.997 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T14:56:51.997 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:51.997 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 ceph-mon[109132]: Deploying daemon osd.3 on smithi099 2023-10-01T14:56:51.997 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:51 smithi099 ceph-mon[109132]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T14:56:52.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:51 smithi135 ceph-mon[109341]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T14:56:52.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:51 smithi135 ceph-mon[109341]: pgmap v67: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:52.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:51 smithi135 ceph-mon[109341]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T14:56:52.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:51 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:52.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:51 smithi135 ceph-mon[109341]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T14:56:52.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:51 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:56:52.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:51 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:56:52.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:51 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:56:52.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:51 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T14:56:52.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:51 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:52.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:51 smithi135 ceph-mon[109341]: Deploying daemon osd.3 on smithi099 2023-10-01T14:56:52.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:51 smithi135 ceph-mon[109341]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T14:56:52.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:51 smithi031 ceph-mon[107152]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T14:56:52.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:51 smithi031 ceph-mon[107152]: pgmap v67: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:52.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:51 smithi031 ceph-mon[107152]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T14:56:52.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:51 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:52.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:51 smithi031 ceph-mon[107152]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T14:56:52.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:51 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T14:56:52.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:51 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T14:56:52.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:51 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T14:56:52.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:51 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T14:56:52.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:51 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:52.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:51 smithi031 ceph-mon[107152]: Deploying daemon osd.3 on smithi099 2023-10-01T14:56:52.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:51 smithi031 ceph-mon[107152]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T14:56:54.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:53 smithi099 ceph-mon[109132]: pgmap v68: 1 pgs: 1 active+clean; 577 KiB data, 83 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:54.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:53 smithi135 ceph-mon[109341]: pgmap v68: 1 pgs: 1 active+clean; 577 KiB data, 83 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:54.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:53 smithi031 ceph-mon[107152]: pgmap v68: 1 pgs: 1 active+clean; 577 KiB data, 83 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:55.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:55 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:55.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:55 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:55.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:55 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:55.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:55 smithi099 ceph-mon[109132]: pgmap v69: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:55.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:55 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:55.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:55 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:55.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:55 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:55.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:55 smithi031 ceph-mon[107152]: pgmap v69: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:55.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:55 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:55.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:55 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:55.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:55 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:55.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:55 smithi135 ceph-mon[109341]: pgmap v69: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2023-10-01T14:56:56.514 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 3 on host 'smithi099' 2023-10-01T14:56:56.905 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:56 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.905 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:56 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.905 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:56 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:56.905 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:56 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:56.905 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:56 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.905 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:56 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.906 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:56 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.906 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:56 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:56.907 DEBUG:teuthology.orchestra.run.smithi099:osd.3> sudo journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-01T14:56:56.909 INFO:tasks.cephadm:Deploying osd.4 on smithi099 with /dev/vg_nvme/lv_2... 2023-10-01T14:56:56.909 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-01T14:56:56.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:56 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:56 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:56 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:56.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:56 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:56.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:56 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:56 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:56 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:56 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:56.940 INFO:journalctl@ceph.osd.3.smithi099.stdout:-- Logs begin at Sun 2023-10-01 14:44:43 UTC. -- 2023-10-01T14:56:56.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:56 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:56 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:56 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:56.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:56 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:56.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:56 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:56 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:56 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:56.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:56 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:56:57.414 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T14:56:57.861 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:57 smithi099 ceph-mon[109132]: pgmap v70: 1 pgs: 1 active+clean; 577 KiB data, 83 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-10-01T14:56:57.861 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:57 smithi099 ceph-mon[109132]: from='osd.3 [v2:172.21.15.99:6808/2445626085,v1:172.21.15.99:6809/2445626085]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T14:56:57.861 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:57 smithi099 ceph-mon[109132]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T14:56:57.861 INFO:journalctl@ceph.osd.3.smithi099.stdout:Oct 01 14:56:57 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-3[117111]: 2023-10-01T14:56:57.635+0000 7fcf13466640 -1 osd.3 0 log_to_monitors true 2023-10-01T14:56:57.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:57 smithi135 ceph-mon[109341]: pgmap v70: 1 pgs: 1 active+clean; 577 KiB data, 83 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-10-01T14:56:57.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:57 smithi135 ceph-mon[109341]: from='osd.3 [v2:172.21.15.99:6808/2445626085,v1:172.21.15.99:6809/2445626085]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T14:56:57.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:57 smithi135 ceph-mon[109341]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T14:56:57.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:57 smithi031 ceph-mon[107152]: pgmap v70: 1 pgs: 1 active+clean; 577 KiB data, 83 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-10-01T14:56:57.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:57 smithi031 ceph-mon[107152]: from='osd.3 [v2:172.21.15.99:6808/2445626085,v1:172.21.15.99:6809/2445626085]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T14:56:57.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:57 smithi031 ceph-mon[107152]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T14:56:58.807 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-01T14:56:58.807 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-01T14:56:58.808 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2023-10-01T14:56:58.808 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2023-10-01T14:56:58.808 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0136364 s, 769 MB/s 2023-10-01T14:56:58.808 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2023-10-01T14:56:59.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi099:vg_nvme/lv_2 2023-10-01T14:56:59.431 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T14:56:59.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T14:56:59.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: osdmap e22: 4 total, 3 up, 4 in 2023-10-01T14:56:59.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:56:59.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: from='osd.3 [v2:172.21.15.99:6808/2445626085,v1:172.21.15.99:6809/2445626085]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:56:59.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:56:59.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 83 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-10-01T14:56:59.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: Detected new or changed devices on smithi099 2023-10-01T14:56:59.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:59.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:59.746 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:59.746 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:59.746 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: Adjusting osd_memory_target on smithi099 to 10604M 2023-10-01T14:56:59.746 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:59.746 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:59.746 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:59.746 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:56:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:59.839 INFO:journalctl@ceph.osd.3.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-3[117111]: 2023-10-01T14:56:59.482+0000 7fcf08ade700 -1 osd.3 0 waiting for initial osdmap 2023-10-01T14:56:59.839 INFO:journalctl@ceph.osd.3.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-3[117111]: 2023-10-01T14:56:59.485+0000 7fcf038d1700 -1 osd.3 23 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T14:56:59.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T14:56:59.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: osdmap e22: 4 total, 3 up, 4 in 2023-10-01T14:56:59.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:56:59.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: from='osd.3 [v2:172.21.15.99:6808/2445626085,v1:172.21.15.99:6809/2445626085]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:56:59.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:56:59.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 83 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-10-01T14:56:59.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: Detected new or changed devices on smithi099 2023-10-01T14:56:59.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:59.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:59.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:59.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:59.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: Adjusting osd_memory_target on smithi099 to 10604M 2023-10-01T14:56:59.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:59.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:59.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:59.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:56:59 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:59.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T14:56:59.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: osdmap e22: 4 total, 3 up, 4 in 2023-10-01T14:56:59.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:56:59.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: from='osd.3 [v2:172.21.15.99:6808/2445626085,v1:172.21.15.99:6809/2445626085]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:56:59.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:56:59.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 83 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-10-01T14:56:59.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: Detected new or changed devices on smithi099 2023-10-01T14:56:59.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:59.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:59.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:59.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:56:59.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: Adjusting osd_memory_target on smithi099 to 10604M 2023-10-01T14:56:59.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:56:59.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:56:59.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:56:59.921 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:56:59 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:00.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:00 smithi099 ceph-mon[109132]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-10-01T14:57:00.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:00 smithi099 ceph-mon[109132]: osdmap e23: 4 total, 3 up, 4 in 2023-10-01T14:57:00.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:57:00.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:57:00.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:00 smithi099 ceph-mon[109132]: osd.3 [v2:172.21.15.99:6808/2445626085,v1:172.21.15.99:6809/2445626085] boot 2023-10-01T14:57:00.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:00 smithi099 ceph-mon[109132]: osdmap e24: 4 total, 4 up, 4 in 2023-10-01T14:57:00.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:57:00.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:57:00.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:00 smithi031 ceph-mon[107152]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-10-01T14:57:00.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:00 smithi031 ceph-mon[107152]: osdmap e23: 4 total, 3 up, 4 in 2023-10-01T14:57:00.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:57:00.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:57:00.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:00 smithi031 ceph-mon[107152]: osd.3 [v2:172.21.15.99:6808/2445626085,v1:172.21.15.99:6809/2445626085] boot 2023-10-01T14:57:00.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:00 smithi031 ceph-mon[107152]: osdmap e24: 4 total, 4 up, 4 in 2023-10-01T14:57:00.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:57:00.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:57:00.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:00 smithi135 ceph-mon[109341]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-10-01T14:57:00.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:00 smithi135 ceph-mon[109341]: osdmap e23: 4 total, 3 up, 4 in 2023-10-01T14:57:00.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:57:00.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:57:00.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:00 smithi135 ceph-mon[109341]: osd.3 [v2:172.21.15.99:6808/2445626085,v1:172.21.15.99:6809/2445626085] boot 2023-10-01T14:57:00.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:00 smithi135 ceph-mon[109341]: osdmap e24: 4 total, 4 up, 4 in 2023-10-01T14:57:00.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:57:00.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T14:57:01.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:01 smithi031 ceph-mon[107152]: purged_snaps scrub starts 2023-10-01T14:57:01.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:01 smithi031 ceph-mon[107152]: purged_snaps scrub ok 2023-10-01T14:57:01.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:01 smithi031 ceph-mon[107152]: from='client.24215 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:57:01.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:01 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:57:01.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:01 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:01.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:01 smithi031 ceph-mon[107152]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 109 MiB used, 358 GiB / 358 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-10-01T14:57:01.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:01 smithi031 ceph-mon[107152]: osdmap e25: 4 total, 4 up, 4 in 2023-10-01T14:57:01.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:01 smithi099 ceph-mon[109132]: purged_snaps scrub starts 2023-10-01T14:57:01.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:01 smithi099 ceph-mon[109132]: purged_snaps scrub ok 2023-10-01T14:57:01.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:01 smithi099 ceph-mon[109132]: from='client.24215 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:57:01.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:01 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:57:01.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:01 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:01.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:01 smithi099 ceph-mon[109132]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 109 MiB used, 358 GiB / 358 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-10-01T14:57:01.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:01 smithi099 ceph-mon[109132]: osdmap e25: 4 total, 4 up, 4 in 2023-10-01T14:57:01.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:01 smithi135 ceph-mon[109341]: purged_snaps scrub starts 2023-10-01T14:57:01.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:01 smithi135 ceph-mon[109341]: purged_snaps scrub ok 2023-10-01T14:57:01.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:01 smithi135 ceph-mon[109341]: from='client.24215 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:57:01.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:01 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:57:01.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:01 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:01.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:01 smithi135 ceph-mon[109341]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 109 MiB used, 358 GiB / 358 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-10-01T14:57:01.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:01 smithi135 ceph-mon[109341]: osdmap e25: 4 total, 4 up, 4 in 2023-10-01T14:57:02.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:02 smithi031 ceph-mon[107152]: from='client.? 172.21.15.99:0/297016483' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "350770c1-1208-4788-afd6-ee4132f85848"}]: dispatch 2023-10-01T14:57:02.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:02 smithi031 ceph-mon[107152]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "350770c1-1208-4788-afd6-ee4132f85848"}]: dispatch 2023-10-01T14:57:02.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:02 smithi031 ceph-mon[107152]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "350770c1-1208-4788-afd6-ee4132f85848"}]': finished 2023-10-01T14:57:02.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:02 smithi031 ceph-mon[107152]: osdmap e26: 5 total, 4 up, 5 in 2023-10-01T14:57:02.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:02.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:02 smithi099 ceph-mon[109132]: from='client.? 172.21.15.99:0/297016483' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "350770c1-1208-4788-afd6-ee4132f85848"}]: dispatch 2023-10-01T14:57:02.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:02 smithi099 ceph-mon[109132]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "350770c1-1208-4788-afd6-ee4132f85848"}]: dispatch 2023-10-01T14:57:02.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:02 smithi099 ceph-mon[109132]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "350770c1-1208-4788-afd6-ee4132f85848"}]': finished 2023-10-01T14:57:02.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:02 smithi099 ceph-mon[109132]: osdmap e26: 5 total, 4 up, 5 in 2023-10-01T14:57:02.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:02 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:02.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:02 smithi135 ceph-mon[109341]: from='client.? 172.21.15.99:0/297016483' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "350770c1-1208-4788-afd6-ee4132f85848"}]: dispatch 2023-10-01T14:57:02.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:02 smithi135 ceph-mon[109341]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "350770c1-1208-4788-afd6-ee4132f85848"}]: dispatch 2023-10-01T14:57:02.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:02 smithi135 ceph-mon[109341]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "350770c1-1208-4788-afd6-ee4132f85848"}]': finished 2023-10-01T14:57:02.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:02 smithi135 ceph-mon[109341]: osdmap e26: 5 total, 4 up, 5 in 2023-10-01T14:57:02.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:02 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:03.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:03 smithi031 ceph-mon[107152]: pgmap v78: 1 pgs: 1 remapped+peering; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-10-01T14:57:03.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:03 smithi031 ceph-mon[107152]: from='client.? 172.21.15.99:0/2645500198' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:57:03.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:03 smithi099 ceph-mon[109132]: pgmap v78: 1 pgs: 1 remapped+peering; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-10-01T14:57:03.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:03 smithi099 ceph-mon[109132]: from='client.? 172.21.15.99:0/2645500198' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:57:03.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:03 smithi135 ceph-mon[109341]: pgmap v78: 1 pgs: 1 remapped+peering; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-10-01T14:57:03.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:03 smithi135 ceph-mon[109341]: from='client.? 172.21.15.99:0/2645500198' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:57:04.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:04 smithi099 ceph-mon[109132]: osdmap e27: 5 total, 4 up, 5 in 2023-10-01T14:57:04.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:04 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:04.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:04 smithi135 ceph-mon[109341]: osdmap e27: 5 total, 4 up, 5 in 2023-10-01T14:57:04.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:04 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:04.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:04 smithi031 ceph-mon[107152]: osdmap e27: 5 total, 4 up, 5 in 2023-10-01T14:57:04.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:04 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:05.795 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:05 smithi099 ceph-mon[109132]: osdmap e28: 5 total, 4 up, 5 in 2023-10-01T14:57:05.795 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:05 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:05.795 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:05 smithi099 ceph-mon[109132]: pgmap v81: 1 pgs: 1 remapped+peering; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-10-01T14:57:05.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:05 smithi135 ceph-mon[109341]: osdmap e28: 5 total, 4 up, 5 in 2023-10-01T14:57:05.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:05 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:05.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:05 smithi135 ceph-mon[109341]: pgmap v81: 1 pgs: 1 remapped+peering; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-10-01T14:57:05.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:05 smithi031 ceph-mon[107152]: osdmap e28: 5 total, 4 up, 5 in 2023-10-01T14:57:05.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:05 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:05.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:05 smithi031 ceph-mon[107152]: pgmap v81: 1 pgs: 1 remapped+peering; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-10-01T14:57:07.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:07 smithi031 ceph-mon[107152]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 109 MiB used, 358 GiB / 358 GiB avail; 107 KiB/s, 0 objects/s recovering 2023-10-01T14:57:07.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T14:57:07.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:07.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:07 smithi031 ceph-mon[107152]: Deploying daemon osd.4 on smithi099 2023-10-01T14:57:07.501 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:07 smithi099 ceph-mon[109132]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 109 MiB used, 358 GiB / 358 GiB avail; 107 KiB/s, 0 objects/s recovering 2023-10-01T14:57:07.501 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T14:57:07.501 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:07.501 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:07 smithi099 ceph-mon[109132]: Deploying daemon osd.4 on smithi099 2023-10-01T14:57:07.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:07 smithi135 ceph-mon[109341]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 109 MiB used, 358 GiB / 358 GiB avail; 107 KiB/s, 0 objects/s recovering 2023-10-01T14:57:07.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T14:57:07.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:07.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:07 smithi135 ceph-mon[109341]: Deploying daemon osd.4 on smithi099 2023-10-01T14:57:09.997 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:09 smithi031 ceph-mon[107152]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 109 MiB used, 358 GiB / 358 GiB avail; 89 KiB/s, 0 objects/s recovering 2023-10-01T14:57:10.090 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:09 smithi099 ceph-mon[109132]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 109 MiB used, 358 GiB / 358 GiB avail; 89 KiB/s, 0 objects/s recovering 2023-10-01T14:57:10.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:09 smithi135 ceph-mon[109341]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 109 MiB used, 358 GiB / 358 GiB avail; 89 KiB/s, 0 objects/s recovering 2023-10-01T14:57:11.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:11 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:11.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:11 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:11.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:11 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:11.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:11 smithi099 ceph-mon[109132]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-01T14:57:11.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:11 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:11.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:11 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:11.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:11 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:11.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:11 smithi135 ceph-mon[109341]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-01T14:57:11.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:11 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:11.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:11 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:11.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:11 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:11.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:11 smithi031 ceph-mon[107152]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-01T14:57:12.226 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 4 on host 'smithi099' 2023-10-01T14:57:12.808 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:12 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:12.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:12 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:12.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:12 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:12.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:12 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:12.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:12 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:12.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:12 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:12.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:12 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:12.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:12 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:12.890 DEBUG:teuthology.orchestra.run.smithi099:osd.4> sudo journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-01T14:57:12.892 INFO:tasks.cephadm:Deploying osd.5 on smithi135 with /dev/vg_nvme/lv_4... 2023-10-01T14:57:12.892 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-01T14:57:12.923 INFO:journalctl@ceph.osd.4.smithi099.stdout:-- Logs begin at Sun 2023-10-01 14:44:43 UTC. -- 2023-10-01T14:57:13.116 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.c/config 2023-10-01T14:57:13.216 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:12 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:13.216 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:12 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:13.216 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:12 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:13.216 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:12 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:13.216 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:12 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:13.216 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:12 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:13.216 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:12 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:13.216 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:12 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:13.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:13.248 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:13.248 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:13.248 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:13.248 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:13.248 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:13.248 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:13.248 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:12 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:13.896 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:13 smithi135 ceph-mon[109341]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-10-01T14:57:13.896 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:13 smithi135 ceph-mon[109341]: from='osd.4 [v2:172.21.15.99:6816/3376064250,v1:172.21.15.99:6817/3376064250]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T14:57:13.896 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:13 smithi135 ceph-mon[109341]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T14:57:14.085 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 14:57:13 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-4[121304]: 2023-10-01T14:57:13.611+0000 7f9a01d01640 -1 osd.4 0 log_to_monitors true 2023-10-01T14:57:14.165 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:13 smithi099 ceph-mon[109132]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-10-01T14:57:14.165 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:13 smithi099 ceph-mon[109132]: from='osd.4 [v2:172.21.15.99:6816/3376064250,v1:172.21.15.99:6817/3376064250]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T14:57:14.165 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:13 smithi099 ceph-mon[109132]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T14:57:14.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:13 smithi031 ceph-mon[107152]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-10-01T14:57:14.260 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:13 smithi031 ceph-mon[107152]: from='osd.4 [v2:172.21.15.99:6816/3376064250,v1:172.21.15.99:6817/3376064250]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T14:57:14.260 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:13 smithi031 ceph-mon[107152]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T14:57:14.360 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-01T14:57:14.361 INFO:teuthology.orchestra.run.smithi135.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-01T14:57:14.361 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10+0 records in 2023-10-01T14:57:14.361 INFO:teuthology.orchestra.run.smithi135.stderr:10+0 records out 2023-10-01T14:57:14.361 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0155595 s, 674 MB/s 2023-10-01T14:57:14.361 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping successful for: 2023-10-01T14:57:14.782 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi135:vg_nvme/lv_4 2023-10-01T14:57:14.996 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.c/config 2023-10-01T14:57:15.345 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: Detected new or changed devices on smithi099 2023-10-01T14:57:15.345 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:15.345 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:15.345 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:15.345 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:15.345 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:15.345 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: Adjusting osd_memory_target on smithi099 to 7069M 2023-10-01T14:57:15.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:15.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:15.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:15.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:15.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-01T14:57:15.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: osdmap e29: 5 total, 4 up, 5 in 2023-10-01T14:57:15.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:15.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: from='osd.4 [v2:172.21.15.99:6816/3376064250,v1:172.21.15.99:6817/3376064250]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:57:15.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:57:15.346 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:15 smithi135 ceph-mon[109341]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-01T14:57:15.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: Detected new or changed devices on smithi099 2023-10-01T14:57:15.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:15.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:15.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:15.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:15.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:15.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: Adjusting osd_memory_target on smithi099 to 7069M 2023-10-01T14:57:15.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:15.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:15.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:15.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:15.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-01T14:57:15.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: osdmap e29: 5 total, 4 up, 5 in 2023-10-01T14:57:15.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:15.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: from='osd.4 [v2:172.21.15.99:6816/3376064250,v1:172.21.15.99:6817/3376064250]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:57:15.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:57:15.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:15 smithi031 ceph-mon[107152]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-01T14:57:15.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: Detected new or changed devices on smithi099 2023-10-01T14:57:15.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:15.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:15.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:15.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:15.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:15.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: Adjusting osd_memory_target on smithi099 to 7069M 2023-10-01T14:57:15.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:15.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:15.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:15.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:15.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-01T14:57:15.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: osdmap e29: 5 total, 4 up, 5 in 2023-10-01T14:57:15.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:15.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: from='osd.4 [v2:172.21.15.99:6816/3376064250,v1:172.21.15.99:6817/3376064250]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:57:15.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-10-01T14:57:15.591 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-mon[109132]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-01T14:57:15.591 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-4[121304]: 2023-10-01T14:57:15.219+0000 7f99f7379700 -1 osd.4 0 waiting for initial osdmap 2023-10-01T14:57:15.591 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 14:57:15 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-4[121304]: 2023-10-01T14:57:15.224+0000 7f99f0969700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T14:57:16.482 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:16 smithi135 ceph-mon[109341]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-10-01T14:57:16.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:16 smithi135 ceph-mon[109341]: osdmap e30: 5 total, 4 up, 5 in 2023-10-01T14:57:16.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:16 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:16.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:16 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:16.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:16 smithi135 ceph-mon[109341]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:57:16.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:16 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:57:16.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:16 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:57:16.485 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:16 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:16.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:16 smithi031 ceph-mon[107152]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-10-01T14:57:16.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:16 smithi031 ceph-mon[107152]: osdmap e30: 5 total, 4 up, 5 in 2023-10-01T14:57:16.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:16 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:16.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:16 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:16.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:16 smithi031 ceph-mon[107152]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:57:16.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:16 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:57:16.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:16 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:57:16.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:16 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:16.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:16 smithi099 ceph-mon[109132]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-10-01T14:57:16.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:16 smithi099 ceph-mon[109132]: osdmap e30: 5 total, 4 up, 5 in 2023-10-01T14:57:16.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:16 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:16.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:16 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:16.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:16 smithi099 ceph-mon[109132]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:57:16.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:16 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:57:16.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:16 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:57:16.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:16 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:17.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:17 smithi031 ceph-mon[107152]: purged_snaps scrub starts 2023-10-01T14:57:17.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:17 smithi031 ceph-mon[107152]: purged_snaps scrub ok 2023-10-01T14:57:17.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:17 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:17.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:17 smithi031 ceph-mon[107152]: osd.4 [v2:172.21.15.99:6816/3376064250,v1:172.21.15.99:6817/3376064250] boot 2023-10-01T14:57:17.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:17 smithi031 ceph-mon[107152]: osdmap e31: 5 total, 5 up, 5 in 2023-10-01T14:57:17.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:17 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:17.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:17 smithi031 ceph-mon[107152]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-10-01T14:57:17.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:17 smithi099 ceph-mon[109132]: purged_snaps scrub starts 2023-10-01T14:57:17.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:17 smithi099 ceph-mon[109132]: purged_snaps scrub ok 2023-10-01T14:57:17.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:17 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:17.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:17 smithi099 ceph-mon[109132]: osd.4 [v2:172.21.15.99:6816/3376064250,v1:172.21.15.99:6817/3376064250] boot 2023-10-01T14:57:17.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:17 smithi099 ceph-mon[109132]: osdmap e31: 5 total, 5 up, 5 in 2023-10-01T14:57:17.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:17 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:17.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:17 smithi099 ceph-mon[109132]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-10-01T14:57:17.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:17 smithi135 ceph-mon[109341]: purged_snaps scrub starts 2023-10-01T14:57:17.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:17 smithi135 ceph-mon[109341]: purged_snaps scrub ok 2023-10-01T14:57:17.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:17 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:17.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:17 smithi135 ceph-mon[109341]: osd.4 [v2:172.21.15.99:6816/3376064250,v1:172.21.15.99:6817/3376064250] boot 2023-10-01T14:57:17.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:17 smithi135 ceph-mon[109341]: osdmap e31: 5 total, 5 up, 5 in 2023-10-01T14:57:17.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:17 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T14:57:17.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:17 smithi135 ceph-mon[109341]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-10-01T14:57:18.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:18 smithi099 ceph-mon[109132]: osdmap e32: 5 total, 5 up, 5 in 2023-10-01T14:57:18.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:18 smithi099 ceph-mon[109132]: from='client.? 172.21.15.135:0/123908957' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d58cd84-2279-4282-a597-bd862b12858f"}]: dispatch 2023-10-01T14:57:18.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:18 smithi099 ceph-mon[109132]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d58cd84-2279-4282-a597-bd862b12858f"}]: dispatch 2023-10-01T14:57:18.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:18 smithi099 ceph-mon[109132]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9d58cd84-2279-4282-a597-bd862b12858f"}]': finished 2023-10-01T14:57:18.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:18 smithi099 ceph-mon[109132]: osdmap e33: 6 total, 5 up, 6 in 2023-10-01T14:57:18.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:18 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:18.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:18 smithi135 ceph-mon[109341]: osdmap e32: 5 total, 5 up, 5 in 2023-10-01T14:57:18.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:18 smithi135 ceph-mon[109341]: from='client.? 172.21.15.135:0/123908957' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d58cd84-2279-4282-a597-bd862b12858f"}]: dispatch 2023-10-01T14:57:18.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:18 smithi135 ceph-mon[109341]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d58cd84-2279-4282-a597-bd862b12858f"}]: dispatch 2023-10-01T14:57:18.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:18 smithi135 ceph-mon[109341]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9d58cd84-2279-4282-a597-bd862b12858f"}]': finished 2023-10-01T14:57:18.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:18 smithi135 ceph-mon[109341]: osdmap e33: 6 total, 5 up, 6 in 2023-10-01T14:57:18.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:18 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:18.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:18 smithi031 ceph-mon[107152]: osdmap e32: 5 total, 5 up, 5 in 2023-10-01T14:57:18.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:18 smithi031 ceph-mon[107152]: from='client.? 172.21.15.135:0/123908957' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d58cd84-2279-4282-a597-bd862b12858f"}]: dispatch 2023-10-01T14:57:18.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:18 smithi031 ceph-mon[107152]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d58cd84-2279-4282-a597-bd862b12858f"}]: dispatch 2023-10-01T14:57:18.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:18 smithi031 ceph-mon[107152]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9d58cd84-2279-4282-a597-bd862b12858f"}]': finished 2023-10-01T14:57:18.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:18 smithi031 ceph-mon[107152]: osdmap e33: 6 total, 5 up, 6 in 2023-10-01T14:57:18.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:18 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:19.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:19 smithi099 ceph-mon[109132]: from='client.? 172.21.15.135:0/543749612' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:57:19.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:19 smithi099 ceph-mon[109132]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:19.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:19 smithi135 ceph-mon[109341]: from='client.? 172.21.15.135:0/543749612' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:57:19.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:19 smithi135 ceph-mon[109341]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:19.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:19 smithi031 ceph-mon[107152]: from='client.? 172.21.15.135:0/543749612' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:57:19.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:19 smithi031 ceph-mon[107152]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:21.892 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:21 smithi135 ceph-mon[109341]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:21.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:21 smithi031 ceph-mon[107152]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:22.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:21 smithi099 ceph-mon[109132]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:22.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:22 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T14:57:22.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:22 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:22.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:22 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T14:57:22.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:22 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:23.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:22 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T14:57:23.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:22 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:23.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:23 smithi135 ceph-mon[109341]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:23.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:23 smithi135 ceph-mon[109341]: Deploying daemon osd.5 on smithi135 2023-10-01T14:57:23.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:23 smithi031 ceph-mon[107152]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:23.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:23 smithi031 ceph-mon[107152]: Deploying daemon osd.5 on smithi135 2023-10-01T14:57:24.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:23 smithi099 ceph-mon[109132]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:24.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:23 smithi099 ceph-mon[109132]: Deploying daemon osd.5 on smithi135 2023-10-01T14:57:25.959 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:25 smithi135 ceph-mon[109341]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:25.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:25 smithi031 ceph-mon[107152]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:26.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:25 smithi099 ceph-mon[109132]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:27.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:27 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:27.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:27 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:27.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:27 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:27.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:27 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:27.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:27 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:27.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:27 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:27.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:27.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:27.515 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:27 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:28.056 INFO:teuthology.orchestra.run.smithi135.stdout:Created osd(s) 5 on host 'smithi135' 2023-10-01T14:57:28.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:28 smithi135 ceph-mon[109341]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:28.179 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:28 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.179 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:28 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.179 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:28 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:28.180 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:28 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:28.180 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:28 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.180 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:28 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.180 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:28 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:28.180 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:28 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:28 smithi099 ceph-mon[109132]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:28.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:28 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:28 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:28 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:28.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:28 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:28.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:28 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:28 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:28 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:28.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:28 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.411 DEBUG:teuthology.orchestra.run.smithi135:osd.5> sudo journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-01T14:57:28.459 INFO:tasks.cephadm:Deploying osd.6 on smithi135 with /dev/vg_nvme/lv_3... 2023-10-01T14:57:28.459 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-01T14:57:28.489 INFO:journalctl@ceph.osd.5.smithi135.stdout:-- Logs begin at Sun 2023-10-01 14:44:43 UTC. -- 2023-10-01T14:57:28.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:28 smithi031 ceph-mon[107152]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:28.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:28 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:28 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:28 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:28.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:28 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:28.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:28 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:28 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:28 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:28.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:28 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:28.944 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.c/config 2023-10-01T14:57:29.419 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 14:57:29 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T14:57:29.295+0000 7fc1a7f4e640 -1 osd.5 0 log_to_monitors true 2023-10-01T14:57:30.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:30 smithi135 ceph-mon[109341]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:30.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:30 smithi135 ceph-mon[109341]: from='osd.5 [v2:172.21.15.135:6800/2282394344,v1:172.21.15.135:6801/2282394344]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T14:57:30.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:30 smithi135 ceph-mon[109341]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T14:57:30.279 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-01T14:57:30.279 INFO:teuthology.orchestra.run.smithi135.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-01T14:57:30.279 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10+0 records in 2023-10-01T14:57:30.279 INFO:teuthology.orchestra.run.smithi135.stderr:10+0 records out 2023-10-01T14:57:30.279 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111662 s, 939 MB/s 2023-10-01T14:57:30.279 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping successful for: 2023-10-01T14:57:30.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:30 smithi099 ceph-mon[109132]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:30.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:30 smithi099 ceph-mon[109132]: from='osd.5 [v2:172.21.15.135:6800/2282394344,v1:172.21.15.135:6801/2282394344]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T14:57:30.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:30 smithi099 ceph-mon[109132]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T14:57:30.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:30 smithi031 ceph-mon[107152]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:30.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:30 smithi031 ceph-mon[107152]: from='osd.5 [v2:172.21.15.135:6800/2282394344,v1:172.21.15.135:6801/2282394344]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T14:57:30.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:30 smithi031 ceph-mon[107152]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T14:57:30.624 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi135:vg_nvme/lv_3 2023-10-01T14:57:30.837 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.c/config 2023-10-01T14:57:31.169 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T14:57:31.090+0000 7fc19d5c6700 -1 osd.5 0 waiting for initial osdmap 2023-10-01T14:57:31.169 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T14:57:31.095+0000 7fc1973b7700 -1 osd.5 35 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T14:57:31.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-mon[109341]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T14:57:31.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-mon[109341]: osdmap e34: 6 total, 5 up, 6 in 2023-10-01T14:57:31.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:31.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-mon[109341]: from='osd.5 [v2:172.21.15.135:6800/2282394344,v1:172.21.15.135:6801/2282394344]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:57:31.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-mon[109341]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:57:31.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:31.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:31.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:31.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:31.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:31.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:31.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:31 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:31.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:31 smithi099 ceph-mon[109132]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T14:57:31.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:31 smithi099 ceph-mon[109132]: osdmap e34: 6 total, 5 up, 6 in 2023-10-01T14:57:31.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:31 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:31.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:31 smithi099 ceph-mon[109132]: from='osd.5 [v2:172.21.15.135:6800/2282394344,v1:172.21.15.135:6801/2282394344]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:57:31.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:31 smithi099 ceph-mon[109132]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:57:31.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:31 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:31.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:31 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:31.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:31 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:31.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:31 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:31.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:31 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:31.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:31 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:31.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:31 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:31.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:31 smithi031 ceph-mon[107152]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T14:57:31.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:31 smithi031 ceph-mon[107152]: osdmap e34: 6 total, 5 up, 6 in 2023-10-01T14:57:31.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:31 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:31.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:31 smithi031 ceph-mon[107152]: from='osd.5 [v2:172.21.15.135:6800/2282394344,v1:172.21.15.135:6801/2282394344]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:57:31.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:31 smithi031 ceph-mon[107152]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:57:31.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:31 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:31.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:31 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:31.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:31 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:31.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:31 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:31.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:31 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:31.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:31 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:31.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:31 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:32.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: Detected new or changed devices on smithi135 2023-10-01T14:57:32.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: Adjusting osd_memory_target on smithi135 to 21209M 2023-10-01T14:57:32.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:32.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-10-01T14:57:32.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: osdmap e35: 6 total, 5 up, 6 in 2023-10-01T14:57:32.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:32.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:32.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: osd.5 [v2:172.21.15.135:6800/2282394344,v1:172.21.15.135:6801/2282394344] boot 2023-10-01T14:57:32.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: osdmap e36: 6 total, 6 up, 6 in 2023-10-01T14:57:32.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:32.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:57:32.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:57:32.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:57:32.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:32 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:32.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: Detected new or changed devices on smithi135 2023-10-01T14:57:32.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: Adjusting osd_memory_target on smithi135 to 21209M 2023-10-01T14:57:32.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:32.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-10-01T14:57:32.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: osdmap e35: 6 total, 5 up, 6 in 2023-10-01T14:57:32.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:32.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:32.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: osd.5 [v2:172.21.15.135:6800/2282394344,v1:172.21.15.135:6801/2282394344] boot 2023-10-01T14:57:32.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: osdmap e36: 6 total, 6 up, 6 in 2023-10-01T14:57:32.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:32.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:57:32.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:57:32.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:57:32.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:32 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:32.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: Detected new or changed devices on smithi135 2023-10-01T14:57:32.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: Adjusting osd_memory_target on smithi135 to 21209M 2023-10-01T14:57:32.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-10-01T14:57:32.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-10-01T14:57:32.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: osdmap e35: 6 total, 5 up, 6 in 2023-10-01T14:57:32.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:32.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:32.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: osd.5 [v2:172.21.15.135:6800/2282394344,v1:172.21.15.135:6801/2282394344] boot 2023-10-01T14:57:32.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: osdmap e36: 6 total, 6 up, 6 in 2023-10-01T14:57:32.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T14:57:32.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:57:32.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:57:32.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:57:32.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:32 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:33.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:33 smithi135 ceph-mon[109341]: purged_snaps scrub starts 2023-10-01T14:57:33.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:33 smithi135 ceph-mon[109341]: purged_snaps scrub ok 2023-10-01T14:57:33.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:33 smithi135 ceph-mon[109341]: osdmap e37: 6 total, 6 up, 6 in 2023-10-01T14:57:33.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:33 smithi135 ceph-mon[109341]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-10-01T14:57:33.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:33 smithi099 ceph-mon[109132]: purged_snaps scrub starts 2023-10-01T14:57:33.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:33 smithi099 ceph-mon[109132]: purged_snaps scrub ok 2023-10-01T14:57:33.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:33 smithi099 ceph-mon[109132]: osdmap e37: 6 total, 6 up, 6 in 2023-10-01T14:57:33.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:33 smithi099 ceph-mon[109132]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-10-01T14:57:33.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:33 smithi031 ceph-mon[107152]: purged_snaps scrub starts 2023-10-01T14:57:33.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:33 smithi031 ceph-mon[107152]: purged_snaps scrub ok 2023-10-01T14:57:33.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:33 smithi031 ceph-mon[107152]: osdmap e37: 6 total, 6 up, 6 in 2023-10-01T14:57:33.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:33 smithi031 ceph-mon[107152]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-10-01T14:57:34.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:34 smithi099 ceph-mon[109132]: osdmap e38: 6 total, 6 up, 6 in 2023-10-01T14:57:34.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:34 smithi099 ceph-mon[109132]: from='client.? 172.21.15.135:0/2606772226' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8d5ee35e-321c-4cf0-a15b-05c8eed2c9eb"}]: dispatch 2023-10-01T14:57:34.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:34 smithi099 ceph-mon[109132]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8d5ee35e-321c-4cf0-a15b-05c8eed2c9eb"}]: dispatch 2023-10-01T14:57:34.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:34 smithi099 ceph-mon[109132]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8d5ee35e-321c-4cf0-a15b-05c8eed2c9eb"}]': finished 2023-10-01T14:57:34.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:34 smithi099 ceph-mon[109132]: osdmap e39: 7 total, 6 up, 7 in 2023-10-01T14:57:34.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:34 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:34.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:34 smithi135 ceph-mon[109341]: osdmap e38: 6 total, 6 up, 6 in 2023-10-01T14:57:34.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:34 smithi135 ceph-mon[109341]: from='client.? 172.21.15.135:0/2606772226' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8d5ee35e-321c-4cf0-a15b-05c8eed2c9eb"}]: dispatch 2023-10-01T14:57:34.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:34 smithi135 ceph-mon[109341]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8d5ee35e-321c-4cf0-a15b-05c8eed2c9eb"}]: dispatch 2023-10-01T14:57:34.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:34 smithi135 ceph-mon[109341]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8d5ee35e-321c-4cf0-a15b-05c8eed2c9eb"}]': finished 2023-10-01T14:57:34.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:34 smithi135 ceph-mon[109341]: osdmap e39: 7 total, 6 up, 7 in 2023-10-01T14:57:34.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:34 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:34.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:34 smithi031 ceph-mon[107152]: osdmap e38: 6 total, 6 up, 6 in 2023-10-01T14:57:34.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:34 smithi031 ceph-mon[107152]: from='client.? 172.21.15.135:0/2606772226' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8d5ee35e-321c-4cf0-a15b-05c8eed2c9eb"}]: dispatch 2023-10-01T14:57:34.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:34 smithi031 ceph-mon[107152]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8d5ee35e-321c-4cf0-a15b-05c8eed2c9eb"}]: dispatch 2023-10-01T14:57:34.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:34 smithi031 ceph-mon[107152]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8d5ee35e-321c-4cf0-a15b-05c8eed2c9eb"}]': finished 2023-10-01T14:57:34.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:34 smithi031 ceph-mon[107152]: osdmap e39: 7 total, 6 up, 7 in 2023-10-01T14:57:34.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:34 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:35.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:35 smithi099 ceph-mon[109132]: from='client.? 172.21.15.135:0/1229699744' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:57:35.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:35 smithi099 ceph-mon[109132]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-10-01T14:57:35.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:35 smithi099 ceph-mon[109132]: osdmap e40: 7 total, 6 up, 7 in 2023-10-01T14:57:35.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:35 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:35.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:35 smithi135 ceph-mon[109341]: from='client.? 172.21.15.135:0/1229699744' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:57:35.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:35 smithi135 ceph-mon[109341]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-10-01T14:57:35.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:35 smithi135 ceph-mon[109341]: osdmap e40: 7 total, 6 up, 7 in 2023-10-01T14:57:35.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:35 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:35.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:35 smithi031 ceph-mon[107152]: from='client.? 172.21.15.135:0/1229699744' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:57:35.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:35 smithi031 ceph-mon[107152]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-10-01T14:57:35.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:35 smithi031 ceph-mon[107152]: osdmap e40: 7 total, 6 up, 7 in 2023-10-01T14:57:35.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:35 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:37.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:37 smithi135 ceph-mon[109341]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 161 MiB used, 536 GiB / 536 GiB avail; 131 KiB/s, 0 objects/s recovering 2023-10-01T14:57:37.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:37 smithi099 ceph-mon[109132]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 161 MiB used, 536 GiB / 536 GiB avail; 131 KiB/s, 0 objects/s recovering 2023-10-01T14:57:37.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:37 smithi031 ceph-mon[107152]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 161 MiB used, 536 GiB / 536 GiB avail; 131 KiB/s, 0 objects/s recovering 2023-10-01T14:57:39.909 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:39 smithi135 ceph-mon[109341]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-01T14:57:39.909 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:39 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T14:57:39.909 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:39 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:39.910 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:39 smithi135 ceph-mon[109341]: Deploying daemon osd.6 on smithi135 2023-10-01T14:57:39.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:39 smithi031 ceph-mon[107152]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-01T14:57:39.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:39 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T14:57:39.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:39 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:39.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:39 smithi031 ceph-mon[107152]: Deploying daemon osd.6 on smithi135 2023-10-01T14:57:40.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:39 smithi099 ceph-mon[109132]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-01T14:57:40.090 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:39 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T14:57:40.090 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:39 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:40.090 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:39 smithi099 ceph-mon[109132]: Deploying daemon osd.6 on smithi135 2023-10-01T14:57:41.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:41 smithi135 ceph-mon[109341]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 78 KiB/s, 0 objects/s recovering 2023-10-01T14:57:41.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:41 smithi031 ceph-mon[107152]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 78 KiB/s, 0 objects/s recovering 2023-10-01T14:57:42.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:41 smithi099 ceph-mon[109132]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 78 KiB/s, 0 objects/s recovering 2023-10-01T14:57:43.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:43.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:43.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:43 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:43.671 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:43 smithi135 ceph-mon[109341]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-10-01T14:57:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:43 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:43.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:43 smithi031 ceph-mon[107152]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-10-01T14:57:43.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:43.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:43.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:43 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:43.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:43 smithi099 ceph-mon[109132]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-10-01T14:57:44.235 INFO:teuthology.orchestra.run.smithi135.stdout:Created osd(s) 6 on host 'smithi135' 2023-10-01T14:57:44.564 DEBUG:teuthology.orchestra.run.smithi135:osd.6> sudo journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-01T14:57:44.567 INFO:tasks.cephadm:Deploying osd.7 on smithi135 with /dev/vg_nvme/lv_2... 2023-10-01T14:57:44.567 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-01T14:57:44.597 INFO:journalctl@ceph.osd.6.smithi135.stdout:-- Logs begin at Sun 2023-10-01 14:44:43 UTC. -- 2023-10-01T14:57:45.090 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.090 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.090 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:45.090 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:45.091 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.091 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.091 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.091 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:44 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:45.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:45.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:45.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.102 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:44 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:45.170 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.c/config 2023-10-01T14:57:45.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:45.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:45.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:45.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:44 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:45.670 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 14:57:45 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T14:57:45.550+0000 7fc021aa5640 -1 osd.6 0 log_to_monitors true 2023-10-01T14:57:45.987 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:45 smithi135 ceph-mon[109341]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-01T14:57:45.987 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:45 smithi135 ceph-mon[109341]: from='osd.6 [v2:172.21.15.135:6808/699159944,v1:172.21.15.135:6809/699159944]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T14:57:45.988 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:45 smithi135 ceph-mon[109341]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T14:57:46.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:45 smithi099 ceph-mon[109132]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-01T14:57:46.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:45 smithi099 ceph-mon[109132]: from='osd.6 [v2:172.21.15.135:6808/699159944,v1:172.21.15.135:6809/699159944]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T14:57:46.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:45 smithi099 ceph-mon[109132]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T14:57:46.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:45 smithi031 ceph-mon[107152]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-01T14:57:46.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:45 smithi031 ceph-mon[107152]: from='osd.6 [v2:172.21.15.135:6808/699159944,v1:172.21.15.135:6809/699159944]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T14:57:46.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:45 smithi031 ceph-mon[107152]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T14:57:46.476 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-01T14:57:46.476 INFO:teuthology.orchestra.run.smithi135.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-01T14:57:46.476 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10+0 records in 2023-10-01T14:57:46.476 INFO:teuthology.orchestra.run.smithi135.stderr:10+0 records out 2023-10-01T14:57:46.476 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0127884 s, 820 MB/s 2023-10-01T14:57:46.477 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping successful for: 2023-10-01T14:57:46.853 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi135:vg_nvme/lv_2 2023-10-01T14:57:47.072 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.c/config 2023-10-01T14:57:47.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-01T14:57:47.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: osdmap e41: 7 total, 6 up, 7 in 2023-10-01T14:57:47.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:47.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: from='osd.6 [v2:172.21.15.135:6808/699159944,v1:172.21.15.135:6809/699159944]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:57:47.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:57:47.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: Detected new or changed devices on smithi135 2023-10-01T14:57:47.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:47.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:47.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:47.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:47.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: Adjusting osd_memory_target on smithi135 to 10604M 2023-10-01T14:57:47.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:47.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:47.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:47.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:47.496 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:47 smithi031 ceph-mon[107152]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-10-01T14:57:47.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-01T14:57:47.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: osdmap e41: 7 total, 6 up, 7 in 2023-10-01T14:57:47.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:47.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: from='osd.6 [v2:172.21.15.135:6808/699159944,v1:172.21.15.135:6809/699159944]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:57:47.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:57:47.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: Detected new or changed devices on smithi135 2023-10-01T14:57:47.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:47.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:47.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:47.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:47.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: Adjusting osd_memory_target on smithi135 to 10604M 2023-10-01T14:57:47.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:47.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:47.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:47.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:47.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:47 smithi099 ceph-mon[109132]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-10-01T14:57:47.618 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T14:57:47.204+0000 7fc01711d700 -1 osd.6 0 waiting for initial osdmap 2023-10-01T14:57:47.618 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T14:57:47.211+0000 7fc012f12700 -1 osd.6 42 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T14:57:47.619 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-01T14:57:47.619 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: osdmap e41: 7 total, 6 up, 7 in 2023-10-01T14:57:47.619 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:47.619 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: from='osd.6 [v2:172.21.15.135:6808/699159944,v1:172.21.15.135:6809/699159944]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:57:47.619 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:57:47.619 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: Detected new or changed devices on smithi135 2023-10-01T14:57:47.619 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:47.619 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:47.619 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:47.619 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:57:47.619 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: Adjusting osd_memory_target on smithi135 to 10604M 2023-10-01T14:57:47.620 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:47.620 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:47.620 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:57:47.620 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:47.620 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:47 smithi135 ceph-mon[109341]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-10-01T14:57:48.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:48 smithi135 ceph-mon[109341]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-10-01T14:57:48.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:48 smithi135 ceph-mon[109341]: osdmap e42: 7 total, 6 up, 7 in 2023-10-01T14:57:48.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:48.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:48.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:57:48.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:57:48.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:48 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:48.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:48 smithi031 ceph-mon[107152]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-10-01T14:57:48.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:48 smithi031 ceph-mon[107152]: osdmap e42: 7 total, 6 up, 7 in 2023-10-01T14:57:48.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:48.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:48.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:57:48.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:57:48.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:48 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:48.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:48 smithi099 ceph-mon[109132]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-10-01T14:57:48.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:48 smithi099 ceph-mon[109132]: osdmap e42: 7 total, 6 up, 7 in 2023-10-01T14:57:48.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:48.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:48.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T14:57:48.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T14:57:48.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:48 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:49.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:49 smithi031 ceph-mon[107152]: purged_snaps scrub starts 2023-10-01T14:57:49.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:49 smithi031 ceph-mon[107152]: purged_snaps scrub ok 2023-10-01T14:57:49.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:49 smithi031 ceph-mon[107152]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:57:49.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:49 smithi031 ceph-mon[107152]: osd.6 [v2:172.21.15.135:6808/699159944,v1:172.21.15.135:6809/699159944] boot 2023-10-01T14:57:49.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:49 smithi031 ceph-mon[107152]: osdmap e43: 7 total, 7 up, 7 in 2023-10-01T14:57:49.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:49 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:49.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:49 smithi031 ceph-mon[107152]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-10-01T14:57:49.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:49 smithi099 ceph-mon[109132]: purged_snaps scrub starts 2023-10-01T14:57:49.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:49 smithi099 ceph-mon[109132]: purged_snaps scrub ok 2023-10-01T14:57:49.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:49 smithi099 ceph-mon[109132]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:57:49.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:49 smithi099 ceph-mon[109132]: osd.6 [v2:172.21.15.135:6808/699159944,v1:172.21.15.135:6809/699159944] boot 2023-10-01T14:57:49.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:49 smithi099 ceph-mon[109132]: osdmap e43: 7 total, 7 up, 7 in 2023-10-01T14:57:49.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:49 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:49.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:49 smithi099 ceph-mon[109132]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-10-01T14:57:49.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:49 smithi135 ceph-mon[109341]: purged_snaps scrub starts 2023-10-01T14:57:49.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:49 smithi135 ceph-mon[109341]: purged_snaps scrub ok 2023-10-01T14:57:49.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:49 smithi135 ceph-mon[109341]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:57:49.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:49 smithi135 ceph-mon[109341]: osd.6 [v2:172.21.15.135:6808/699159944,v1:172.21.15.135:6809/699159944] boot 2023-10-01T14:57:49.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:49 smithi135 ceph-mon[109341]: osdmap e43: 7 total, 7 up, 7 in 2023-10-01T14:57:49.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:49 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T14:57:49.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:49 smithi135 ceph-mon[109341]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-10-01T14:57:50.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:50 smithi031 ceph-mon[107152]: osdmap e44: 7 total, 7 up, 7 in 2023-10-01T14:57:50.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:50 smithi031 ceph-mon[107152]: from='client.? 172.21.15.135:0/4109017769' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "beb2487d-145f-4129-90f4-f9b62ca2bd59"}]: dispatch 2023-10-01T14:57:50.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:50 smithi031 ceph-mon[107152]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "beb2487d-145f-4129-90f4-f9b62ca2bd59"}]: dispatch 2023-10-01T14:57:50.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:50 smithi031 ceph-mon[107152]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "beb2487d-145f-4129-90f4-f9b62ca2bd59"}]': finished 2023-10-01T14:57:50.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:50 smithi031 ceph-mon[107152]: osdmap e45: 8 total, 7 up, 8 in 2023-10-01T14:57:50.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:50 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:57:50.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:50 smithi099 ceph-mon[109132]: osdmap e44: 7 total, 7 up, 7 in 2023-10-01T14:57:50.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:50 smithi099 ceph-mon[109132]: from='client.? 172.21.15.135:0/4109017769' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "beb2487d-145f-4129-90f4-f9b62ca2bd59"}]: dispatch 2023-10-01T14:57:50.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:50 smithi099 ceph-mon[109132]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "beb2487d-145f-4129-90f4-f9b62ca2bd59"}]: dispatch 2023-10-01T14:57:50.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:50 smithi099 ceph-mon[109132]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "beb2487d-145f-4129-90f4-f9b62ca2bd59"}]': finished 2023-10-01T14:57:50.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:50 smithi099 ceph-mon[109132]: osdmap e45: 8 total, 7 up, 8 in 2023-10-01T14:57:50.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:50 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:57:50.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:50 smithi135 ceph-mon[109341]: osdmap e44: 7 total, 7 up, 7 in 2023-10-01T14:57:50.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:50 smithi135 ceph-mon[109341]: from='client.? 172.21.15.135:0/4109017769' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "beb2487d-145f-4129-90f4-f9b62ca2bd59"}]: dispatch 2023-10-01T14:57:50.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:50 smithi135 ceph-mon[109341]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "beb2487d-145f-4129-90f4-f9b62ca2bd59"}]: dispatch 2023-10-01T14:57:50.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:50 smithi135 ceph-mon[109341]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "beb2487d-145f-4129-90f4-f9b62ca2bd59"}]': finished 2023-10-01T14:57:50.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:50 smithi135 ceph-mon[109341]: osdmap e45: 8 total, 7 up, 8 in 2023-10-01T14:57:50.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:50 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:57:51.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:51 smithi031 ceph-mon[107152]: from='client.? 172.21.15.135:0/2840562950' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:57:51.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:51 smithi031 ceph-mon[107152]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:51.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:51 smithi099 ceph-mon[109132]: from='client.? 172.21.15.135:0/2840562950' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:57:51.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:51 smithi099 ceph-mon[109132]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:51.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:51 smithi135 ceph-mon[109341]: from='client.? 172.21.15.135:0/2840562950' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T14:57:51.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:51 smithi135 ceph-mon[109341]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:53.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:53 smithi135 ceph-mon[109341]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:53.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:53 smithi031 ceph-mon[107152]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:54.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:53 smithi099 ceph-mon[109132]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:55.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:55 smithi031 ceph-mon[107152]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:55.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:55 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T14:57:55.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:55 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:55.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:55 smithi031 ceph-mon[107152]: Deploying daemon osd.7 on smithi135 2023-10-01T14:57:56.000 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:55 smithi135 ceph-mon[109341]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:56.000 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:55 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T14:57:56.000 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:55 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:56.000 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:55 smithi135 ceph-mon[109341]: Deploying daemon osd.7 on smithi135 2023-10-01T14:57:56.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:55 smithi099 ceph-mon[109132]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:56.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:55 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T14:57:56.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:55 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:57:56.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:55 smithi099 ceph-mon[109132]: Deploying daemon osd.7 on smithi135 2023-10-01T14:57:57.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:57 smithi099 ceph-mon[109132]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:57.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:57 smithi135 ceph-mon[109341]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:57.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:57 smithi031 ceph-mon[107152]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:59.527 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:59 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:59.527 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:59 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:59.527 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:59 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:59.528 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:57:59 smithi135 ceph-mon[109341]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:59.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:59 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:59.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:59 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:59.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:59 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:59.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:57:59 smithi099 ceph-mon[109132]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:57:59.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:59.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:57:59.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:59 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:57:59.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:57:59 smithi031 ceph-mon[107152]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:58:00.429 INFO:teuthology.orchestra.run.smithi135.stdout:Created osd(s) 7 on host 'smithi135' 2023-10-01T14:58:00.807 DEBUG:teuthology.orchestra.run.smithi135:osd.7> sudo journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-01T14:58:00.809 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-01T14:58:00.809 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-01T14:58:00.840 INFO:journalctl@ceph.osd.7.smithi135.stdout:-- Logs begin at Sun 2023-10-01 14:44:43 UTC. -- 2023-10-01T14:58:01.006 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.006 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.006 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:58:01.006 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:58:01.007 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.007 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.007 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:58:01.007 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:00 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.061 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:01.187 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.187 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.187 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:58:01.187 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:58:01.188 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.188 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.188 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:58:01.188 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:00 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:58:01.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:58:01.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:58:01.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:00 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:01.890 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 14:58:01 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T14:58:01.767+0000 7f6e6f23c640 -1 osd.7 0 log_to_monitors true 2023-10-01T14:58:01.953 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:01 smithi031 ceph-mon[107152]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:58:01.954 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:01 smithi031 ceph-mon[107152]: from='osd.7 [v2:172.21.15.135:6816/862967489,v1:172.21.15.135:6817/862967489]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T14:58:01.954 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:01 smithi031 ceph-mon[107152]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T14:58:02.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:01 smithi135 ceph-mon[109341]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:58:02.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:01 smithi135 ceph-mon[109341]: from='osd.7 [v2:172.21.15.135:6816/862967489,v1:172.21.15.135:6817/862967489]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T14:58:02.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:01 smithi135 ceph-mon[109341]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T14:58:02.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:01 smithi099 ceph-mon[109132]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:58:02.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:01 smithi099 ceph-mon[109132]: from='osd.7 [v2:172.21.15.135:6816/862967489,v1:172.21.15.135:6817/862967489]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T14:58:02.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:01 smithi099 ceph-mon[109132]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T14:58:02.369 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:58:02.816 INFO:teuthology.orchestra.run.smithi031.stdout:{"epoch":45,"num_osds":8,"num_up_osds":7,"osd_up_since":1696172268,"num_in_osds":8,"osd_in_since":1696172269,"num_remapped_pgs":0} 2023-10-01T14:58:02.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/4167598605' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-01T14:58:02.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-01T14:58:02.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: osdmap e46: 8 total, 7 up, 8 in 2023-10-01T14:58:02.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:58:02.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='osd.7 [v2:172.21.15.135:6816/862967489,v1:172.21.15.135:6817/862967489]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:58:02.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:58:02.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:02.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:02.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:58:02.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:58:02.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:58:02.996 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:02.996 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:58:02.996 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:58:02.996 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:02 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:03.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/4167598605' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-01T14:58:03.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-01T14:58:03.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: osdmap e46: 8 total, 7 up, 8 in 2023-10-01T14:58:03.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:58:03.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='osd.7 [v2:172.21.15.135:6816/862967489,v1:172.21.15.135:6817/862967489]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:58:03.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:58:03.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:03.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:03.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:58:03.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:58:03.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:58:03.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:03.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:58:03.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:58:03.171 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:02 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:03.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/4167598605' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-01T14:58:03.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-01T14:58:03.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: osdmap e46: 8 total, 7 up, 8 in 2023-10-01T14:58:03.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:58:03.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='osd.7 [v2:172.21.15.135:6816/862967489,v1:172.21.15.135:6817/862967489]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:58:03.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2023-10-01T14:58:03.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:03.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:03.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:58:03.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:58:03.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T14:58:03.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:03.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:58:03.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:58:03.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:02 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:58:03.817 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-01T14:58:03.893 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 14:58:03 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T14:58:03.418+0000 7f6e648b4700 -1 osd.7 0 waiting for initial osdmap 2023-10-01T14:58:03.893 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 14:58:03 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T14:58:03.425+0000 7f6e606a9700 -1 osd.7 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T14:58:04.069 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:04.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:03 smithi135 ceph-mon[109341]: Detected new or changed devices on smithi135 2023-10-01T14:58:04.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:03 smithi135 ceph-mon[109341]: Adjusting osd_memory_target on smithi135 to 7069M 2023-10-01T14:58:04.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:03 smithi135 ceph-mon[109341]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:58:04.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:03 smithi135 ceph-mon[109341]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-10-01T14:58:04.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:03 smithi135 ceph-mon[109341]: osdmap e47: 8 total, 7 up, 8 in 2023-10-01T14:58:04.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:03 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:58:04.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:03 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:58:04.203 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:03 smithi031 ceph-mon[107152]: Detected new or changed devices on smithi135 2023-10-01T14:58:04.203 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:03 smithi031 ceph-mon[107152]: Adjusting osd_memory_target on smithi135 to 7069M 2023-10-01T14:58:04.203 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:03 smithi031 ceph-mon[107152]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:58:04.203 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:03 smithi031 ceph-mon[107152]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-10-01T14:58:04.203 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:03 smithi031 ceph-mon[107152]: osdmap e47: 8 total, 7 up, 8 in 2023-10-01T14:58:04.203 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:03 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:58:04.203 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:03 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:58:04.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:03 smithi099 ceph-mon[109132]: Detected new or changed devices on smithi135 2023-10-01T14:58:04.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:03 smithi099 ceph-mon[109132]: Adjusting osd_memory_target on smithi135 to 7069M 2023-10-01T14:58:04.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:03 smithi099 ceph-mon[109132]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-10-01T14:58:04.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:03 smithi099 ceph-mon[109132]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2023-10-01T14:58:04.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:03 smithi099 ceph-mon[109132]: osdmap e47: 8 total, 7 up, 8 in 2023-10-01T14:58:04.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:03 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:58:04.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:03 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:58:05.359 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:58:05.683 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:05 smithi031 ceph-mon[107152]: purged_snaps scrub starts 2023-10-01T14:58:05.683 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:05 smithi031 ceph-mon[107152]: purged_snaps scrub ok 2023-10-01T14:58:05.683 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:05 smithi031 ceph-mon[107152]: osd.7 [v2:172.21.15.135:6816/862967489,v1:172.21.15.135:6817/862967489] boot 2023-10-01T14:58:05.683 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:05 smithi031 ceph-mon[107152]: osdmap e48: 8 total, 8 up, 8 in 2023-10-01T14:58:05.683 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:05 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:58:05.683 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:05 smithi031 ceph-mon[107152]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:05.683 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:05 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/3089585686' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-01T14:58:05.748 INFO:teuthology.orchestra.run.smithi031.stdout:{"epoch":48,"num_osds":8,"num_up_osds":8,"osd_up_since":1696172284,"num_in_osds":8,"osd_in_since":1696172269,"num_remapped_pgs":0} 2023-10-01T14:58:05.748 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T14:58:05.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:05 smithi099 ceph-mon[109132]: purged_snaps scrub starts 2023-10-01T14:58:05.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:05 smithi099 ceph-mon[109132]: purged_snaps scrub ok 2023-10-01T14:58:05.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:05 smithi099 ceph-mon[109132]: osd.7 [v2:172.21.15.135:6816/862967489,v1:172.21.15.135:6817/862967489] boot 2023-10-01T14:58:05.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:05 smithi099 ceph-mon[109132]: osdmap e48: 8 total, 8 up, 8 in 2023-10-01T14:58:05.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:05 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:58:05.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:05 smithi099 ceph-mon[109132]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:05.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:05 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/3089585686' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-01T14:58:05.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:05 smithi135 ceph-mon[109341]: purged_snaps scrub starts 2023-10-01T14:58:05.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:05 smithi135 ceph-mon[109341]: purged_snaps scrub ok 2023-10-01T14:58:05.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:05 smithi135 ceph-mon[109341]: osd.7 [v2:172.21.15.135:6816/862967489,v1:172.21.15.135:6817/862967489] boot 2023-10-01T14:58:05.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:05 smithi135 ceph-mon[109341]: osdmap e48: 8 total, 8 up, 8 in 2023-10-01T14:58:05.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:05 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T14:58:05.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:05 smithi135 ceph-mon[109341]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:05.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:05 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/3089585686' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-01T14:58:05.997 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:06.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:06 smithi031 ceph-mon[107152]: osdmap e49: 8 total, 8 up, 8 in 2023-10-01T14:58:06.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:06 smithi099 ceph-mon[109132]: osdmap e49: 8 total, 8 up, 8 in 2023-10-01T14:58:06.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:06 smithi135 ceph-mon[109341]: osdmap e49: 8 total, 8 up, 8 in 2023-10-01T14:58:07.220 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:58:07.220 INFO:teuthology.orchestra.run.smithi031.stdout:{"epoch":49,"fsid":"466018a4-606a-11ee-8db5-212e2dc638e7","created":"2023-10-01T14:54:11.180877+0000","modified":"2023-10-01T14:58:05.415884+0000","last_up_change":"2023-10-01T14:58:04.413847+0000","last_in_change":"2023-10-01T14:57:49.905172+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-01T14:56:45.136104+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"1d3a06bc-29dd-4824-847c-173f7a20a4d3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6802","nonce":3666492266},{"type":"v1","addr":"172.21.15.31:6803","nonce":3666492266}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6804","nonce":3666492266},{"type":"v1","addr":"172.21.15.31:6805","nonce":3666492266}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6808","nonce":3666492266},{"type":"v1","addr":"172.21.15.31:6809","nonce":3666492266}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6806","nonce":3666492266},{"type":"v1","addr":"172.21.15.31:6807","nonce":3666492266}]},"public_addr":"172.21.15.31:6803/3666492266","cluster_addr":"172.21.15.31:6805/3666492266","heartbeat_back_addr":"172.21.15.31:6809/3666492266","heartbeat_front_addr":"172.21.15.31:6807/3666492266","state":["exists","up"]},{"osd":1,"uuid":"db968f7c-d7ed-447b-b74b-bf21a6ba0b4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6810","nonce":3064465530},{"type":"v1","addr":"172.21.15.31:6811","nonce":3064465530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6812","nonce":3064465530},{"type":"v1","addr":"172.21.15.31:6813","nonce":3064465530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6816","nonce":3064465530},{"type":"v1","addr":"172.21.15.31:6817","nonce":3064465530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6814","nonce":3064465530},{"type":"v1","addr":"172.21.15.31:6815","nonce":3064465530}]},"public_addr":"172.21.15.31:6811/3064465530","cluster_addr":"172.21.15.31:6813/3064465530","heartbeat_back_addr":"172.21.15.31:6817/3064465530","heartbeat_front_addr":"172.21.15.31:6815/3064465530","state":["exists","up"]},{"osd":2,"uuid":"7eb06643-5689-4ee7-94d9-58a39433c2ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":19,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":563131629},{"type":"v1","addr":"172.21.15.99:6801","nonce":563131629}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":563131629},{"type":"v1","addr":"172.21.15.99:6803","nonce":563131629}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":563131629},{"type":"v1","addr":"172.21.15.99:6807","nonce":563131629}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":563131629},{"type":"v1","addr":"172.21.15.99:6805","nonce":563131629}]},"public_addr":"172.21.15.99:6801/563131629","cluster_addr":"172.21.15.99:6803/563131629","heartbeat_back_addr":"172.21.15.99:6807/563131629","heartbeat_front_addr":"172.21.15.99:6805/563131629","state":["exists","up"]},{"osd":3,"uuid":"0a17c4e0-6068-4031-b08a-1bec022f3752","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":24,"up_thru":39,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2445626085},{"type":"v1","addr":"172.21.15.99:6809","nonce":2445626085}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":2445626085},{"type":"v1","addr":"172.21.15.99:6811","nonce":2445626085}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":2445626085},{"type":"v1","addr":"172.21.15.99:6815","nonce":2445626085}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":2445626085},{"type":"v1","addr":"172.21.15.99:6813","nonce":2445626085}]},"public_addr":"172.21.15.99:6809/2445626085","cluster_addr":"172.21.15.99:6811/2445626085","heartbeat_back_addr":"172.21.15.99:6815/2445626085","heartbeat_front_addr":"172.21.15.99:6813/2445626085","state":["exists","up"]},{"osd":4,"uuid":"350770c1-1208-4788-afd6-ee4132f85848","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3376064250},{"type":"v1","addr":"172.21.15.99:6817","nonce":3376064250}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":3376064250},{"type":"v1","addr":"172.21.15.99:6819","nonce":3376064250}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":3376064250},{"type":"v1","addr":"172.21.15.99:6823","nonce":3376064250}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":3376064250},{"type":"v1","addr":"172.21.15.99:6821","nonce":3376064250}]},"public_addr":"172.21.15.99:6817/3376064250","cluster_addr":"172.21.15.99:6819/3376064250","heartbeat_back_addr":"172.21.15.99:6823/3376064250","heartbeat_front_addr":"172.21.15.99:6821/3376064250","state":["exists","up"]},{"osd":5,"uuid":"9d58cd84-2279-4282-a597-bd862b12858f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6800","nonce":2282394344},{"type":"v1","addr":"172.21.15.135:6801","nonce":2282394344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":2282394344},{"type":"v1","addr":"172.21.15.135:6803","nonce":2282394344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":2282394344},{"type":"v1","addr":"172.21.15.135:6807","nonce":2282394344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":2282394344},{"type":"v1","addr":"172.21.15.135:6805","nonce":2282394344}]},"public_addr":"172.21.15.135:6801/2282394344","cluster_addr":"172.21.15.135:6803/2282394344","heartbeat_back_addr":"172.21.15.135:6807/2282394344","heartbeat_front_addr":"172.21.15.135:6805/2282394344","state":["exists","up"]},{"osd":6,"uuid":"8d5ee35e-321c-4cf0-a15b-05c8eed2c9eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":699159944},{"type":"v1","addr":"172.21.15.135:6809","nonce":699159944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":699159944},{"type":"v1","addr":"172.21.15.135:6811","nonce":699159944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":699159944},{"type":"v1","addr":"172.21.15.135:6815","nonce":699159944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":699159944},{"type":"v1","addr":"172.21.15.135:6813","nonce":699159944}]},"public_addr":"172.21.15.135:6809/699159944","cluster_addr":"172.21.15.135:6811/699159944","heartbeat_back_addr":"172.21.15.135:6815/699159944","heartbeat_front_addr":"172.21.15.135:6813/699159944","state":["exists","up"]},{"osd":7,"uuid":"beb2487d-145f-4129-90f4-f9b62ca2bd59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":862967489},{"type":"v1","addr":"172.21.15.135:6817","nonce":862967489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":862967489},{"type":"v1","addr":"172.21.15.135:6819","nonce":862967489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":862967489},{"type":"v1","addr":"172.21.15.135:6823","nonce":862967489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":862967489},{"type":"v1","addr":"172.21.15.135:6821","nonce":862967489}]},"public_addr":"172.21.15.135:6817/862967489","cluster_addr":"172.21.15.135:6819/862967489","heartbeat_back_addr":"172.21.15.135:6823/862967489","heartbeat_front_addr":"172.21.15.135:6821/862967489","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:56:08.891180+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:56:26.954922+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:56:42.711593+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:56:58.623488+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:57:14.609717+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:57:30.288788+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:57:46.553898+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:58:02.789608+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.31:0/2877523079":"2023-10-02T14:54:58.602532+0000","172.21.15.31:0/775807865":"2023-10-02T14:54:58.602532+0000","172.21.15.31:0/3118401886":"2023-10-02T14:54:36.973478+0000","172.21.15.31:6801/3068836595":"2023-10-02T14:54:36.973478+0000","172.21.15.31:6800/554886134":"2023-10-02T14:54:58.602532+0000","172.21.15.31:0/3757320562":"2023-10-02T14:54:36.973478+0000","172.21.15.31:6801/554886134":"2023-10-02T14:54:58.602532+0000","172.21.15.31:6800/3068836595":"2023-10-02T14:54:36.973478+0000","172.21.15.31:0/799820806":"2023-10-02T14:54:36.973478+0000","172.21.15.31:0/3602921581":"2023-10-02T14:54:58.602532+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T14:58:07.658 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:07 smithi031 ceph-mon[107152]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:07.658 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:07 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/4112762156' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T14:58:07.685 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-10-01T14:56:45.136104+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 7.889999866485596, 'score_stable': 7.889999866485596, 'optimal_score': 0.3799999952316284, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2023-10-01T14:58:07.686 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-10-01T14:58:07.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:07 smithi099 ceph-mon[109132]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:07.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:07 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/4112762156' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T14:58:07.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:07 smithi135 ceph-mon[109341]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:07.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:07 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/4112762156' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T14:58:07.934 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:09.306 INFO:teuthology.orchestra.run.smithi031.stdout:pg_num: 1 2023-10-01T14:58:09.707 INFO:tasks.cephadm:Setting up client nodes... 2023-10-01T14:58:09.708 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-01T14:58:09.952 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:09.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:09 smithi031 ceph-mon[107152]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:09.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:09 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/2561586149' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-01T14:58:10.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:09 smithi099 ceph-mon[109132]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:10.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:09 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/2561586149' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-01T14:58:10.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:09 smithi135 ceph-mon[109341]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:10.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:09 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/2561586149' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-01T14:58:11.547 INFO:teuthology.orchestra.run.smithi031.stdout:[client.0] 2023-10-01T14:58:11.547 INFO:teuthology.orchestra.run.smithi031.stdout: key = AQADiRll3T83IBAA0fPr4WhP5ziB+72GuxF5zw== 2023-10-01T14:58:11.873 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:11 smithi031 ceph-mon[107152]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:11.873 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:11 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/686336861' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:11.874 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:11 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/686336861' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T14:58:11.954 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2023-10-01T14:58:11.954 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-01T14:58:11.954 DEBUG:teuthology.orchestra.run.smithi031:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-01T14:58:12.004 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-01T14:58:12.039 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:11 smithi099 ceph-mon[109132]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:12.039 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:11 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/686336861' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:12.039 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:11 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/686336861' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T14:58:12.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:11 smithi135 ceph-mon[109341]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:12.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:11 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/686336861' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:12.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:11 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/686336861' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T14:58:12.233 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T14:58:13.416 INFO:teuthology.orchestra.run.smithi099.stdout:[client.1] 2023-10-01T14:58:13.416 INFO:teuthology.orchestra.run.smithi099.stdout: key = AQAFiRllMkKBGBAAMLv3PkCFLEDACblHWCxtwA== 2023-10-01T14:58:13.789 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-10-01T14:58:13.789 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-01T14:58:13.789 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-01T14:58:13.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:13 smithi099 ceph-mon[109132]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:13.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:13 smithi099 ceph-mon[109132]: from='client.? 172.21.15.99:0/2764874578' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:13.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:13 smithi099 ceph-mon[109132]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:13.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:13 smithi099 ceph-mon[109132]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T14:58:13.869 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.2 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-01T14:58:13.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:13 smithi031 ceph-mon[107152]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:13.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:13 smithi031 ceph-mon[107152]: from='client.? 172.21.15.99:0/2764874578' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:13.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:13 smithi031 ceph-mon[107152]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:13.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:13 smithi031 ceph-mon[107152]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T14:58:14.104 INFO:teuthology.orchestra.run.smithi135.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.c/config 2023-10-01T14:58:14.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:13 smithi135 ceph-mon[109341]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:14.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:13 smithi135 ceph-mon[109341]: from='client.? 172.21.15.99:0/2764874578' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:14.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:13 smithi135 ceph-mon[109341]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:14.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:13 smithi135 ceph-mon[109341]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T14:58:15.306 INFO:teuthology.orchestra.run.smithi135.stdout:[client.2] 2023-10-01T14:58:15.306 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAHiRllHhftERAAziAN/YD1BIEP91qfFO+Uow== 2023-10-01T14:58:15.682 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2023-10-01T14:58:15.682 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/ceph/ceph.client.2.keyring 2023-10-01T14:58:15.682 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod 0644 /etc/ceph/ceph.client.2.keyring 2023-10-01T14:58:15.724 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-01T14:58:15.724 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-01T14:58:15.724 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-01T14:58:15.973 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:15.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:15 smithi031 ceph-mon[107152]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:15.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:15 smithi031 ceph-mon[107152]: from='client.? 172.21.15.135:0/1719882873' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.2", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:15.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:15 smithi031 ceph-mon[107152]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.2", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:15.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:15 smithi031 ceph-mon[107152]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.2", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T14:58:16.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:15 smithi099 ceph-mon[109132]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:16.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:15 smithi099 ceph-mon[109132]: from='client.? 172.21.15.135:0/1719882873' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.2", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:16.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:15 smithi099 ceph-mon[109132]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.2", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:16.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:15 smithi099 ceph-mon[109132]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.2", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T14:58:16.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:15 smithi135 ceph-mon[109341]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:16.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:15 smithi135 ceph-mon[109341]: from='client.? 172.21.15.135:0/1719882873' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.2", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:16.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:15 smithi135 ceph-mon[109341]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.2", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T14:58:16.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:15 smithi135 ceph-mon[109341]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.2", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T14:58:17.406 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:58:17.584 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:17 smithi031 ceph-mon[107152]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:17.588 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:17 smithi099 ceph-mon[109132]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:17.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:17 smithi135 ceph-mon[109341]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:17.797 INFO:teuthology.orchestra.run.smithi031.stdout:{"epoch":14,"active_gid":14152,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6800","nonce":948143798},{"type":"v1","addr":"172.21.15.31:6801","nonce":948143798}]},"active_addr":"172.21.15.31:6801/948143798","active_change":"2023-10-01T14:54:58.602700+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[],"modules":["cephadm","dashboard","iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.3","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.31:8443/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":3,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.31:0","nonce":379498900}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.31:0","nonce":4239188841}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.31:0","nonce":1746190669}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.31:0","nonce":4278893662}]}]} 2023-10-01T14:58:17.799 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-01T14:58:17.799 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-01T14:58:17.800 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T14:58:18.045 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:18.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:18 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/3203510941' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T14:58:18.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:18 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/3203510941' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T14:58:18.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:18 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/3203510941' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T14:58:19.393 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:58:19.393 INFO:teuthology.orchestra.run.smithi031.stdout:{"epoch":49,"fsid":"466018a4-606a-11ee-8db5-212e2dc638e7","created":"2023-10-01T14:54:11.180877+0000","modified":"2023-10-01T14:58:05.415884+0000","last_up_change":"2023-10-01T14:58:04.413847+0000","last_in_change":"2023-10-01T14:57:49.905172+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-01T14:56:45.136104+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"1d3a06bc-29dd-4824-847c-173f7a20a4d3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6802","nonce":3666492266},{"type":"v1","addr":"172.21.15.31:6803","nonce":3666492266}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6804","nonce":3666492266},{"type":"v1","addr":"172.21.15.31:6805","nonce":3666492266}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6808","nonce":3666492266},{"type":"v1","addr":"172.21.15.31:6809","nonce":3666492266}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6806","nonce":3666492266},{"type":"v1","addr":"172.21.15.31:6807","nonce":3666492266}]},"public_addr":"172.21.15.31:6803/3666492266","cluster_addr":"172.21.15.31:6805/3666492266","heartbeat_back_addr":"172.21.15.31:6809/3666492266","heartbeat_front_addr":"172.21.15.31:6807/3666492266","state":["exists","up"]},{"osd":1,"uuid":"db968f7c-d7ed-447b-b74b-bf21a6ba0b4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6810","nonce":3064465530},{"type":"v1","addr":"172.21.15.31:6811","nonce":3064465530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6812","nonce":3064465530},{"type":"v1","addr":"172.21.15.31:6813","nonce":3064465530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6816","nonce":3064465530},{"type":"v1","addr":"172.21.15.31:6817","nonce":3064465530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6814","nonce":3064465530},{"type":"v1","addr":"172.21.15.31:6815","nonce":3064465530}]},"public_addr":"172.21.15.31:6811/3064465530","cluster_addr":"172.21.15.31:6813/3064465530","heartbeat_back_addr":"172.21.15.31:6817/3064465530","heartbeat_front_addr":"172.21.15.31:6815/3064465530","state":["exists","up"]},{"osd":2,"uuid":"7eb06643-5689-4ee7-94d9-58a39433c2ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":19,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":563131629},{"type":"v1","addr":"172.21.15.99:6801","nonce":563131629}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":563131629},{"type":"v1","addr":"172.21.15.99:6803","nonce":563131629}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":563131629},{"type":"v1","addr":"172.21.15.99:6807","nonce":563131629}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":563131629},{"type":"v1","addr":"172.21.15.99:6805","nonce":563131629}]},"public_addr":"172.21.15.99:6801/563131629","cluster_addr":"172.21.15.99:6803/563131629","heartbeat_back_addr":"172.21.15.99:6807/563131629","heartbeat_front_addr":"172.21.15.99:6805/563131629","state":["exists","up"]},{"osd":3,"uuid":"0a17c4e0-6068-4031-b08a-1bec022f3752","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":24,"up_thru":39,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2445626085},{"type":"v1","addr":"172.21.15.99:6809","nonce":2445626085}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":2445626085},{"type":"v1","addr":"172.21.15.99:6811","nonce":2445626085}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":2445626085},{"type":"v1","addr":"172.21.15.99:6815","nonce":2445626085}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":2445626085},{"type":"v1","addr":"172.21.15.99:6813","nonce":2445626085}]},"public_addr":"172.21.15.99:6809/2445626085","cluster_addr":"172.21.15.99:6811/2445626085","heartbeat_back_addr":"172.21.15.99:6815/2445626085","heartbeat_front_addr":"172.21.15.99:6813/2445626085","state":["exists","up"]},{"osd":4,"uuid":"350770c1-1208-4788-afd6-ee4132f85848","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3376064250},{"type":"v1","addr":"172.21.15.99:6817","nonce":3376064250}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":3376064250},{"type":"v1","addr":"172.21.15.99:6819","nonce":3376064250}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":3376064250},{"type":"v1","addr":"172.21.15.99:6823","nonce":3376064250}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":3376064250},{"type":"v1","addr":"172.21.15.99:6821","nonce":3376064250}]},"public_addr":"172.21.15.99:6817/3376064250","cluster_addr":"172.21.15.99:6819/3376064250","heartbeat_back_addr":"172.21.15.99:6823/3376064250","heartbeat_front_addr":"172.21.15.99:6821/3376064250","state":["exists","up"]},{"osd":5,"uuid":"9d58cd84-2279-4282-a597-bd862b12858f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6800","nonce":2282394344},{"type":"v1","addr":"172.21.15.135:6801","nonce":2282394344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":2282394344},{"type":"v1","addr":"172.21.15.135:6803","nonce":2282394344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":2282394344},{"type":"v1","addr":"172.21.15.135:6807","nonce":2282394344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":2282394344},{"type":"v1","addr":"172.21.15.135:6805","nonce":2282394344}]},"public_addr":"172.21.15.135:6801/2282394344","cluster_addr":"172.21.15.135:6803/2282394344","heartbeat_back_addr":"172.21.15.135:6807/2282394344","heartbeat_front_addr":"172.21.15.135:6805/2282394344","state":["exists","up"]},{"osd":6,"uuid":"8d5ee35e-321c-4cf0-a15b-05c8eed2c9eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":699159944},{"type":"v1","addr":"172.21.15.135:6809","nonce":699159944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":699159944},{"type":"v1","addr":"172.21.15.135:6811","nonce":699159944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":699159944},{"type":"v1","addr":"172.21.15.135:6815","nonce":699159944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":699159944},{"type":"v1","addr":"172.21.15.135:6813","nonce":699159944}]},"public_addr":"172.21.15.135:6809/699159944","cluster_addr":"172.21.15.135:6811/699159944","heartbeat_back_addr":"172.21.15.135:6815/699159944","heartbeat_front_addr":"172.21.15.135:6813/699159944","state":["exists","up"]},{"osd":7,"uuid":"beb2487d-145f-4129-90f4-f9b62ca2bd59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":862967489},{"type":"v1","addr":"172.21.15.135:6817","nonce":862967489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":862967489},{"type":"v1","addr":"172.21.15.135:6819","nonce":862967489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":862967489},{"type":"v1","addr":"172.21.15.135:6823","nonce":862967489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":862967489},{"type":"v1","addr":"172.21.15.135:6821","nonce":862967489}]},"public_addr":"172.21.15.135:6817/862967489","cluster_addr":"172.21.15.135:6819/862967489","heartbeat_back_addr":"172.21.15.135:6823/862967489","heartbeat_front_addr":"172.21.15.135:6821/862967489","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:56:08.891180+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:56:26.954922+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:56:42.711593+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:56:58.623488+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:57:14.609717+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:57:30.288788+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:57:46.553898+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:58:02.789608+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.31:0/2877523079":"2023-10-02T14:54:58.602532+0000","172.21.15.31:0/775807865":"2023-10-02T14:54:58.602532+0000","172.21.15.31:0/3118401886":"2023-10-02T14:54:36.973478+0000","172.21.15.31:6801/3068836595":"2023-10-02T14:54:36.973478+0000","172.21.15.31:6800/554886134":"2023-10-02T14:54:58.602532+0000","172.21.15.31:0/3757320562":"2023-10-02T14:54:36.973478+0000","172.21.15.31:6801/554886134":"2023-10-02T14:54:58.602532+0000","172.21.15.31:6800/3068836595":"2023-10-02T14:54:36.973478+0000","172.21.15.31:0/799820806":"2023-10-02T14:54:36.973478+0000","172.21.15.31:0/3602921581":"2023-10-02T14:54:58.602532+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T14:58:19.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:19 smithi099 ceph-mon[109132]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:19.642 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:19 smithi031 ceph-mon[107152]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:19.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:19 smithi135 ceph-mon[109341]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:19.829 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-01T14:58:19.829 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T14:58:20.077 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:20.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:20 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/1451589712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T14:58:20.588 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:20 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/1451589712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T14:58:20.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:20 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/1451589712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T14:58:21.264 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:58:21.264 INFO:teuthology.orchestra.run.smithi031.stdout:{"epoch":49,"fsid":"466018a4-606a-11ee-8db5-212e2dc638e7","created":"2023-10-01T14:54:11.180877+0000","modified":"2023-10-01T14:58:05.415884+0000","last_up_change":"2023-10-01T14:58:04.413847+0000","last_in_change":"2023-10-01T14:57:49.905172+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-01T14:56:45.136104+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"1d3a06bc-29dd-4824-847c-173f7a20a4d3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6802","nonce":3666492266},{"type":"v1","addr":"172.21.15.31:6803","nonce":3666492266}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6804","nonce":3666492266},{"type":"v1","addr":"172.21.15.31:6805","nonce":3666492266}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6808","nonce":3666492266},{"type":"v1","addr":"172.21.15.31:6809","nonce":3666492266}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6806","nonce":3666492266},{"type":"v1","addr":"172.21.15.31:6807","nonce":3666492266}]},"public_addr":"172.21.15.31:6803/3666492266","cluster_addr":"172.21.15.31:6805/3666492266","heartbeat_back_addr":"172.21.15.31:6809/3666492266","heartbeat_front_addr":"172.21.15.31:6807/3666492266","state":["exists","up"]},{"osd":1,"uuid":"db968f7c-d7ed-447b-b74b-bf21a6ba0b4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6810","nonce":3064465530},{"type":"v1","addr":"172.21.15.31:6811","nonce":3064465530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6812","nonce":3064465530},{"type":"v1","addr":"172.21.15.31:6813","nonce":3064465530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6816","nonce":3064465530},{"type":"v1","addr":"172.21.15.31:6817","nonce":3064465530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6814","nonce":3064465530},{"type":"v1","addr":"172.21.15.31:6815","nonce":3064465530}]},"public_addr":"172.21.15.31:6811/3064465530","cluster_addr":"172.21.15.31:6813/3064465530","heartbeat_back_addr":"172.21.15.31:6817/3064465530","heartbeat_front_addr":"172.21.15.31:6815/3064465530","state":["exists","up"]},{"osd":2,"uuid":"7eb06643-5689-4ee7-94d9-58a39433c2ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":19,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":563131629},{"type":"v1","addr":"172.21.15.99:6801","nonce":563131629}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":563131629},{"type":"v1","addr":"172.21.15.99:6803","nonce":563131629}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":563131629},{"type":"v1","addr":"172.21.15.99:6807","nonce":563131629}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":563131629},{"type":"v1","addr":"172.21.15.99:6805","nonce":563131629}]},"public_addr":"172.21.15.99:6801/563131629","cluster_addr":"172.21.15.99:6803/563131629","heartbeat_back_addr":"172.21.15.99:6807/563131629","heartbeat_front_addr":"172.21.15.99:6805/563131629","state":["exists","up"]},{"osd":3,"uuid":"0a17c4e0-6068-4031-b08a-1bec022f3752","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":24,"up_thru":39,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2445626085},{"type":"v1","addr":"172.21.15.99:6809","nonce":2445626085}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":2445626085},{"type":"v1","addr":"172.21.15.99:6811","nonce":2445626085}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":2445626085},{"type":"v1","addr":"172.21.15.99:6815","nonce":2445626085}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":2445626085},{"type":"v1","addr":"172.21.15.99:6813","nonce":2445626085}]},"public_addr":"172.21.15.99:6809/2445626085","cluster_addr":"172.21.15.99:6811/2445626085","heartbeat_back_addr":"172.21.15.99:6815/2445626085","heartbeat_front_addr":"172.21.15.99:6813/2445626085","state":["exists","up"]},{"osd":4,"uuid":"350770c1-1208-4788-afd6-ee4132f85848","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3376064250},{"type":"v1","addr":"172.21.15.99:6817","nonce":3376064250}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":3376064250},{"type":"v1","addr":"172.21.15.99:6819","nonce":3376064250}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":3376064250},{"type":"v1","addr":"172.21.15.99:6823","nonce":3376064250}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":3376064250},{"type":"v1","addr":"172.21.15.99:6821","nonce":3376064250}]},"public_addr":"172.21.15.99:6817/3376064250","cluster_addr":"172.21.15.99:6819/3376064250","heartbeat_back_addr":"172.21.15.99:6823/3376064250","heartbeat_front_addr":"172.21.15.99:6821/3376064250","state":["exists","up"]},{"osd":5,"uuid":"9d58cd84-2279-4282-a597-bd862b12858f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6800","nonce":2282394344},{"type":"v1","addr":"172.21.15.135:6801","nonce":2282394344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":2282394344},{"type":"v1","addr":"172.21.15.135:6803","nonce":2282394344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":2282394344},{"type":"v1","addr":"172.21.15.135:6807","nonce":2282394344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":2282394344},{"type":"v1","addr":"172.21.15.135:6805","nonce":2282394344}]},"public_addr":"172.21.15.135:6801/2282394344","cluster_addr":"172.21.15.135:6803/2282394344","heartbeat_back_addr":"172.21.15.135:6807/2282394344","heartbeat_front_addr":"172.21.15.135:6805/2282394344","state":["exists","up"]},{"osd":6,"uuid":"8d5ee35e-321c-4cf0-a15b-05c8eed2c9eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":699159944},{"type":"v1","addr":"172.21.15.135:6809","nonce":699159944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":699159944},{"type":"v1","addr":"172.21.15.135:6811","nonce":699159944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":699159944},{"type":"v1","addr":"172.21.15.135:6815","nonce":699159944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":699159944},{"type":"v1","addr":"172.21.15.135:6813","nonce":699159944}]},"public_addr":"172.21.15.135:6809/699159944","cluster_addr":"172.21.15.135:6811/699159944","heartbeat_back_addr":"172.21.15.135:6815/699159944","heartbeat_front_addr":"172.21.15.135:6813/699159944","state":["exists","up"]},{"osd":7,"uuid":"beb2487d-145f-4129-90f4-f9b62ca2bd59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":862967489},{"type":"v1","addr":"172.21.15.135:6817","nonce":862967489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":862967489},{"type":"v1","addr":"172.21.15.135:6819","nonce":862967489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":862967489},{"type":"v1","addr":"172.21.15.135:6823","nonce":862967489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":862967489},{"type":"v1","addr":"172.21.15.135:6821","nonce":862967489}]},"public_addr":"172.21.15.135:6817/862967489","cluster_addr":"172.21.15.135:6819/862967489","heartbeat_back_addr":"172.21.15.135:6823/862967489","heartbeat_front_addr":"172.21.15.135:6821/862967489","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:56:08.891180+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:56:26.954922+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:56:42.711593+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:56:58.623488+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:57:14.609717+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:57:30.288788+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:57:46.553898+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T14:58:02.789608+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.31:0/2877523079":"2023-10-02T14:54:58.602532+0000","172.21.15.31:0/775807865":"2023-10-02T14:54:58.602532+0000","172.21.15.31:0/3118401886":"2023-10-02T14:54:36.973478+0000","172.21.15.31:6801/3068836595":"2023-10-02T14:54:36.973478+0000","172.21.15.31:6800/554886134":"2023-10-02T14:54:58.602532+0000","172.21.15.31:0/3757320562":"2023-10-02T14:54:36.973478+0000","172.21.15.31:6801/554886134":"2023-10-02T14:54:58.602532+0000","172.21.15.31:6800/3068836595":"2023-10-02T14:54:36.973478+0000","172.21.15.31:0/799820806":"2023-10-02T14:54:36.973478+0000","172.21.15.31:0/3602921581":"2023-10-02T14:54:58.602532+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T14:58:21.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:21 smithi031 ceph-mon[107152]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:21.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:21 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/1715497718' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T14:58:21.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:21 smithi099 ceph-mon[109132]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:21.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:21 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/1715497718' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T14:58:21.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:21 smithi135 ceph-mon[109341]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:21.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:21 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/1715497718' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T14:58:21.687 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-01T14:58:21.688 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-01T14:58:21.688 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-01T14:58:21.688 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-01T14:58:21.689 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-01T14:58:21.689 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-01T14:58:21.690 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-01T14:58:21.690 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-01T14:58:22.112 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:22.338 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:22.637 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:22.941 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:23.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:23 smithi031 ceph-mon[107152]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:24.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:23 smithi099 ceph-mon[109132]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:24.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:23 smithi135 ceph-mon[109341]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:24.887 INFO:teuthology.orchestra.run.smithi031.stdout:77309411351 2023-10-01T14:58:24.887 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-01T14:58:24.898 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:24.991 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:25.039 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:25.049 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:25.978 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:25 smithi031 ceph-mon[107152]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:26.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:25 smithi099 ceph-mon[109132]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:26.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:25 smithi135 ceph-mon[109341]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:27.186 INFO:teuthology.orchestra.run.smithi031.stdout:103079215124 2023-10-01T14:58:27.186 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-01T14:58:27.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:27 smithi031 ceph-mon[107152]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:28.412 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:27 smithi135 ceph-mon[109341]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:28.412 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:27 smithi099 ceph-mon[109132]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:28.413 INFO:teuthology.orchestra.run.smithi031.stdout:154618822669 2023-10-01T14:58:28.413 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-01T14:58:29.614 INFO:teuthology.orchestra.run.smithi031.stdout:206158430215 2023-10-01T14:58:29.614 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-01T14:58:29.768 INFO:teuthology.orchestra.run.smithi031.stdout:34359738399 2023-10-01T14:58:29.768 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-01T14:58:29.882 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:29 smithi031 ceph-mon[107152]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:29.883 INFO:teuthology.orchestra.run.smithi031.stdout:55834574874 2023-10-01T14:58:29.883 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-01T14:58:30.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:29 smithi099 ceph-mon[109132]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:30.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:29 smithi135 ceph-mon[109341]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:30.267 INFO:teuthology.orchestra.run.smithi031.stdout:133143986194 2023-10-01T14:58:30.268 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-01T14:58:30.287 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:30.290 INFO:teuthology.orchestra.run.smithi031.stdout:184683593739 2023-10-01T14:58:30.291 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-01T14:58:30.338 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:30.345 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:30.785 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:30.976 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:31.881 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:31 smithi031 ceph-mon[107152]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:32.075 INFO:teuthology.orchestra.run.smithi031.stdout:55834574875 2023-10-01T14:58:32.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:31 smithi099 ceph-mon[109132]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:32.103 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:32.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:31 smithi135 ceph-mon[109341]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:32.728 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:32 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/903651230' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T14:58:32.892 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:33.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:32 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/903651230' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T14:58:33.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:32 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/903651230' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T14:58:33.536 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574874 got 55834574875 for osd.1 2023-10-01T14:58:33.612 DEBUG:teuthology.parallel:result is None 2023-10-01T14:58:33.738 INFO:teuthology.orchestra.run.smithi031.stdout:34359738400 2023-10-01T14:58:33.841 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:33 smithi031 ceph-mon[107152]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:34.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:33 smithi099 ceph-mon[109132]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:34.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:33 smithi135 ceph-mon[109341]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:34.443 INFO:teuthology.orchestra.run.smithi031.stdout:77309411353 2023-10-01T14:58:34.707 INFO:teuthology.orchestra.run.smithi031.stdout:206158430216 2023-10-01T14:58:34.739 INFO:teuthology.orchestra.run.smithi031.stdout:103079215125 2023-10-01T14:58:34.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:34 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/2490028934' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T14:58:35.023 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:34 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/4000174753' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T14:58:35.023 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:34 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/428679278' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T14:58:35.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:34 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/2490028934' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T14:58:35.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:34 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/4000174753' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T14:58:35.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:34 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/428679278' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T14:58:35.121 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411351 got 77309411353 for osd.2 2023-10-01T14:58:35.121 DEBUG:teuthology.parallel:result is None 2023-10-01T14:58:35.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:34 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/2490028934' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T14:58:35.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:34 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/4000174753' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T14:58:35.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:34 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/428679278' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T14:58:35.247 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:35.866 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738399 got 34359738400 for osd.0 2023-10-01T14:58:35.866 DEBUG:teuthology.parallel:result is None 2023-10-01T14:58:35.911 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430215 got 206158430216 for osd.7 2023-10-01T14:58:35.911 DEBUG:teuthology.parallel:result is None 2023-10-01T14:58:35.922 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:35 smithi031 ceph-mon[107152]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:35.922 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:35 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/1264889175' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T14:58:36.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:35 smithi099 ceph-mon[109132]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:36.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:35 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/1264889175' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T14:58:36.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:35 smithi135 ceph-mon[109341]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:36.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:35 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/1264889175' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T14:58:36.265 INFO:tasks.cephadm.ceph_manager.ceph:need seq 103079215124 got 103079215125 for osd.3 2023-10-01T14:58:36.265 DEBUG:teuthology.parallel:result is None 2023-10-01T14:58:36.344 INFO:teuthology.orchestra.run.smithi031.stdout:154618822671 2023-10-01T14:58:36.638 INFO:teuthology.orchestra.run.smithi031.stdout:133143986195 2023-10-01T14:58:37.008 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822669 got 154618822671 for osd.5 2023-10-01T14:58:37.008 DEBUG:teuthology.parallel:result is None 2023-10-01T14:58:37.318 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986194 got 133143986195 for osd.4 2023-10-01T14:58:37.318 DEBUG:teuthology.parallel:result is None 2023-10-01T14:58:37.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:37 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/2481220671' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T14:58:37.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:37 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/727243140' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T14:58:37.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:37 smithi099 ceph-mon[109132]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:37.627 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:37 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/2481220671' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T14:58:37.628 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:37 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/727243140' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T14:58:37.628 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:37 smithi031 ceph-mon[107152]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:37.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:37 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/2481220671' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T14:58:37.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:37 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/727243140' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T14:58:37.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:37 smithi135 ceph-mon[109341]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:38.078 INFO:teuthology.orchestra.run.smithi031.stdout:184683593741 2023-10-01T14:58:38.469 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:38 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/4055943579' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T14:58:38.492 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593739 got 184683593741 for osd.6 2023-10-01T14:58:38.492 DEBUG:teuthology.parallel:result is None 2023-10-01T14:58:38.493 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-01T14:58:38.493 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T14:58:38.588 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:38 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/4055943579' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T14:58:38.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:38 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/4055943579' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T14:58:38.739 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:39.586 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:39 smithi031 ceph-mon[107152]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:39.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:39 smithi099 ceph-mon[109132]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:39.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:39 smithi135 ceph-mon[109341]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:40.027 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:58:40.029 INFO:teuthology.orchestra.run.smithi031.stderr:dumped all 2023-10-01T14:58:40.335 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:40 smithi031 ceph-mon[107152]: from='client.14517 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T14:58:40.426 INFO:teuthology.orchestra.run.smithi031.stdout:{"pg_ready":true,"pg_map":{"version":149,"stamp":"2023-10-01T14:58:38.648844+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":221608,"kb_used_data":5420,"kb_used_omap":13,"kb_used_meta":216050,"kb_avail":749739608,"statfs":{"total":767960285184,"available":767733358592,"internally_reserved":0,"allocated":5550080,"data_stored":3306088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14248,"internal_metadata":221235288},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002450"},"pg_stats":[{"pgid":"1.0","version":"21'137","reported_seq":241,"reported_epoch":48,"state":"active+clean","last_fresh":"2023-10-01T14:58:04.699715+0000","last_change":"2023-10-01T14:57:34.879440+0000","last_active":"2023-10-01T14:58:04.699715+0000","last_peered":"2023-10-01T14:58:04.699715+0000","last_clean":"2023-10-01T14:58:04.699715+0000","last_became_active":"2023-10-01T14:57:34.879028+0000","last_became_peered":"2023-10-01T14:57:34.879028+0000","last_unstale":"2023-10-01T14:58:04.699715+0000","last_undegraded":"2023-10-01T14:58:04.699715+0000","last_fullsized":"2023-10-01T14:58:04.699715+0000","mapping_epoch":39,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":40,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T14:56:45.404345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T14:56:45.404345+0000","last_clean_scrub_stamp":"2023-10-01T14:56:45.404345+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-03T02:03:18.943502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430217,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27480,"kb_used_data":460,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717672,"statfs":{"total":95995035648,"available":95966896128,"internally_reserved":0,"allocated":471040,"data_stored":191873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.439}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76600000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.83199999999999996}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.79100000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73099999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67000000000000004}]}]},{"osd":6,"up_from":43,"seq":184683593741,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27488,"kb_used_data":460,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717664,"statfs":{"total":95995035648,"available":95966887936,"internally_reserved":0,"allocated":471040,"data_stored":191873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57899999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65600000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59799999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40600000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48299999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574876,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27484,"kb_used_data":460,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717668,"statfs":{"total":95995035648,"available":95966892032,"internally_reserved":0,"allocated":471040,"data_stored":191873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 14:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.35899999999999999,"15min":0.35699999999999998},"min":{"1min":0.26900000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.38300000000000001,"15min":0.38500000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.505,"5min":0.505,"15min":0.505},"last":0.39100000000000001}]},{"osd":2,"last update":"Sun Oct 1 14:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.65000000000000002}]},{"osd":3,"last update":"Sun Oct 1 14:58:01 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.62},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.68999999999999995}]},{"osd":4,"last update":"Sun Oct 1 14:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.63100000000000001}]},{"osd":5,"last update":"Sun Oct 1 14:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.74299999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.90800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.86199999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738401,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27488,"kb_used_data":460,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717664,"statfs":{"total":95995035648,"available":95966887936,"internally_reserved":0,"allocated":471040,"data_stored":191873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 14:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"min":{"1min":0.26700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.48099999999999998,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.38,"15min":0.38200000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.29499999999999998}]},{"osd":2,"last update":"Sun Oct 1 14:57:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.627}]},{"osd":3,"last update":"Sun Oct 1 14:58:02 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.72299999999999998}]},{"osd":4,"last update":"Sun Oct 1 14:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.59199999999999997}]},{"osd":5,"last update":"Sun Oct 1 14:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.66700000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64100000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411354,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28064,"kb_used_data":1040,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717088,"statfs":{"total":95995035648,"available":95966298112,"internally_reserved":0,"allocated":1064960,"data_stored":782241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 14:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.47199999999999998}]},{"osd":1,"last update":"Sun Oct 1 14:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.505},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.45600000000000002}]},{"osd":3,"last update":"Sun Oct 1 14:58:02 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.437}]},{"osd":4,"last update":"Sun Oct 1 14:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.38400000000000001}]},{"osd":5,"last update":"Sun Oct 1 14:58:34 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.57299999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53600000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]}]},{"osd":3,"up_from":24,"seq":103079215126,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28064,"kb_used_data":1040,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717088,"statfs":{"total":95995035648,"available":95966298112,"internally_reserved":0,"allocated":1064960,"data_stored":782241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 14:58:05 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.495},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.42699999999999999}]},{"osd":1,"last update":"Sun Oct 1 14:58:05 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.47099999999999997}]},{"osd":2,"last update":"Sun Oct 1 14:58:05 2023","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.309}]},{"osd":4,"last update":"Sun Oct 1 14:58:21 2023","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.39300000000000002}]},{"osd":5,"last update":"Sun Oct 1 14:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.59099999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55800000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51300000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986196,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27480,"kb_used_data":460,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717672,"statfs":{"total":95995035648,"available":95966896128,"internally_reserved":0,"allocated":471040,"data_stored":191873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 14:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.38}]},{"osd":1,"last update":"Sun Oct 1 14:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.438},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.44900000000000001}]},{"osd":2,"last update":"Sun Oct 1 14:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.33600000000000002}]},{"osd":3,"last update":"Sun Oct 1 14:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.58199999999999996}]},{"osd":5,"last update":"Sun Oct 1 14:58:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.53200000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63500000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822672,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28060,"kb_used_data":1040,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717092,"statfs":{"total":95995035648,"available":95966302208,"internally_reserved":0,"allocated":1064960,"data_stored":782241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 14:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":3.4409999999999998,"5min":3.4409999999999998,"15min":3.4409999999999998},"last":0.378},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.355,"5min":3.355,"15min":3.355},"last":0.42699999999999999}]},{"osd":1,"last update":"Sun Oct 1 14:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.4199999999999999,"5min":3.4199999999999999,"15min":3.4199999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.48199999999999998}]},{"osd":2,"last update":"Sun Oct 1 14:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.63},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.65500000000000003}]},{"osd":3,"last update":"Sun Oct 1 14:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.69499999999999995}]},{"osd":4,"last update":"Sun Oct 1 14:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.53000000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T14:58:40.428 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T14:58:40.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:40 smithi099 ceph-mon[109132]: from='client.14517 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T14:58:40.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:40 smithi135 ceph-mon[109341]: from='client.14517 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T14:58:40.675 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:41.491 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:41 smithi031 ceph-mon[107152]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:41.588 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:41 smithi099 ceph-mon[109132]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:41.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:41 smithi135 ceph-mon[109341]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:41.923 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:58:41.925 INFO:teuthology.orchestra.run.smithi031.stderr:dumped all 2023-10-01T14:58:42.341 INFO:teuthology.orchestra.run.smithi031.stdout:{"pg_ready":true,"pg_map":{"version":150,"stamp":"2023-10-01T14:58:40.649151+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":221608,"kb_used_data":5420,"kb_used_omap":13,"kb_used_meta":216050,"kb_avail":749739608,"statfs":{"total":767960285184,"available":767733358592,"internally_reserved":0,"allocated":5550080,"data_stored":3306088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14248,"internal_metadata":221235288},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002377"},"pg_stats":[{"pgid":"1.0","version":"21'137","reported_seq":241,"reported_epoch":48,"state":"active+clean","last_fresh":"2023-10-01T14:58:04.699715+0000","last_change":"2023-10-01T14:57:34.879440+0000","last_active":"2023-10-01T14:58:04.699715+0000","last_peered":"2023-10-01T14:58:04.699715+0000","last_clean":"2023-10-01T14:58:04.699715+0000","last_became_active":"2023-10-01T14:57:34.879028+0000","last_became_peered":"2023-10-01T14:57:34.879028+0000","last_unstale":"2023-10-01T14:58:04.699715+0000","last_undegraded":"2023-10-01T14:58:04.699715+0000","last_fullsized":"2023-10-01T14:58:04.699715+0000","mapping_epoch":39,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":40,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T14:56:45.404345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T14:56:45.404345+0000","last_clean_scrub_stamp":"2023-10-01T14:56:45.404345+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-03T02:03:18.943502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430218,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27480,"kb_used_data":460,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717672,"statfs":{"total":95995035648,"available":95966896128,"internally_reserved":0,"allocated":471040,"data_stored":191873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.433}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60199999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47699999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.371}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46500000000000002}]}]},{"osd":6,"up_from":43,"seq":184683593742,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27488,"kb_used_data":460,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717664,"statfs":{"total":95995035648,"available":95966887936,"internally_reserved":0,"allocated":471040,"data_stored":191873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61299999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.624}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58799999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67700000000000005}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70999999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48999999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56899999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574877,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27484,"kb_used_data":460,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717668,"statfs":{"total":95995035648,"available":95966892032,"internally_reserved":0,"allocated":471040,"data_stored":191873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 14:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.35899999999999999,"15min":0.35699999999999998},"min":{"1min":0.26900000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.38300000000000001,"15min":0.38500000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.505,"5min":0.505,"15min":0.505},"last":0.29199999999999998}]},{"osd":2,"last update":"Sun Oct 1 14:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.55300000000000005}]},{"osd":3,"last update":"Sun Oct 1 14:58:01 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.59999999999999998}]},{"osd":4,"last update":"Sun Oct 1 14:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.53900000000000003}]},{"osd":5,"last update":"Sun Oct 1 14:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.46700000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58799999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.626}]}]},{"osd":0,"up_from":8,"seq":34359738401,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27488,"kb_used_data":460,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717664,"statfs":{"total":95995035648,"available":95966887936,"internally_reserved":0,"allocated":471040,"data_stored":191873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 14:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"min":{"1min":0.26700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.48099999999999998,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.38,"15min":0.38200000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.29499999999999998}]},{"osd":2,"last update":"Sun Oct 1 14:57:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.627}]},{"osd":3,"last update":"Sun Oct 1 14:58:02 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.72299999999999998}]},{"osd":4,"last update":"Sun Oct 1 14:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.59199999999999997}]},{"osd":5,"last update":"Sun Oct 1 14:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.66700000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64100000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411355,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28064,"kb_used_data":1040,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717088,"statfs":{"total":95995035648,"available":95966298112,"internally_reserved":0,"allocated":1064960,"data_stored":782241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 14:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.58599999999999997}]},{"osd":1,"last update":"Sun Oct 1 14:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.55500000000000005}]},{"osd":3,"last update":"Sun Oct 1 14:58:02 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.5}]},{"osd":4,"last update":"Sun Oct 1 14:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.442},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.48399999999999999}]},{"osd":5,"last update":"Sun Oct 1 14:58:34 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.68400000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70699999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60099999999999998}]}]},{"osd":3,"up_from":24,"seq":103079215127,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28064,"kb_used_data":1040,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717088,"statfs":{"total":95995035648,"available":95966298112,"internally_reserved":0,"allocated":1064960,"data_stored":782241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 14:58:05 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.40400000000000003}]},{"osd":1,"last update":"Sun Oct 1 14:58:05 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.5}]},{"osd":2,"last update":"Sun Oct 1 14:58:05 2023","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.56999999999999995}]},{"osd":4,"last update":"Sun Oct 1 14:58:21 2023","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.46899999999999997}]},{"osd":5,"last update":"Sun Oct 1 14:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.62},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.54700000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63100000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986196,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27480,"kb_used_data":460,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717672,"statfs":{"total":95995035648,"available":95966896128,"internally_reserved":0,"allocated":471040,"data_stored":191873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 14:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.38}]},{"osd":1,"last update":"Sun Oct 1 14:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.438},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.44900000000000001}]},{"osd":2,"last update":"Sun Oct 1 14:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.33600000000000002}]},{"osd":3,"last update":"Sun Oct 1 14:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.58199999999999996}]},{"osd":5,"last update":"Sun Oct 1 14:58:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.53200000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63500000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822672,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28060,"kb_used_data":1040,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717092,"statfs":{"total":95995035648,"available":95966302208,"internally_reserved":0,"allocated":1064960,"data_stored":782241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 14:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":3.4409999999999998,"5min":3.4409999999999998,"15min":3.4409999999999998},"last":0.378},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.355,"5min":3.355,"15min":3.355},"last":0.42699999999999999}]},{"osd":1,"last update":"Sun Oct 1 14:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.4199999999999999,"5min":3.4199999999999999,"15min":3.4199999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.48199999999999998}]},{"osd":2,"last update":"Sun Oct 1 14:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.63},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.65500000000000003}]},{"osd":3,"last update":"Sun Oct 1 14:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.69499999999999995}]},{"osd":4,"last update":"Sun Oct 1 14:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.53000000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T14:58:42.343 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-01T14:58:42.343 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-01T14:58:42.343 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-01T14:58:42.343 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-01T14:58:42.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:42 smithi099 ceph-mon[109132]: from='client.14523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T14:58:42.589 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:42.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:42 smithi135 ceph-mon[109341]: from='client.14523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T14:58:42.721 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:42 smithi031 ceph-mon[107152]: from='client.14523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T14:58:43.451 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:43 smithi031 ceph-mon[107152]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:43.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:43 smithi099 ceph-mon[109132]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:43.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:43 smithi135 ceph-mon[109341]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:43.955 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:58:43.955 INFO:teuthology.orchestra.run.smithi031.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-01T14:58:44.350 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-01T14:58:44.351 INFO:tasks.cephadm:Setup complete, yielding 2023-10-01T14:58:44.351 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-01T14:58:44.359 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi031.front.sepia.ceph.com 2023-10-01T14:58:44.366 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch status' 2023-10-01T14:58:44.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:44 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/264361797' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T14:58:44.616 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:44.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:44 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/264361797' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T14:58:44.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:44 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/264361797' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T14:58:45.488 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:45 smithi031 ceph-mon[107152]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:45.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:45 smithi099 ceph-mon[109132]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:45.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:45 smithi135 ceph-mon[109341]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:45.890 INFO:teuthology.orchestra.run.smithi031.stdout:Backend: cephadm 2023-10-01T14:58:45.891 INFO:teuthology.orchestra.run.smithi031.stdout:Available: Yes 2023-10-01T14:58:45.891 INFO:teuthology.orchestra.run.smithi031.stdout:Paused: No 2023-10-01T14:58:46.333 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch ps' 2023-10-01T14:58:46.582 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:46.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:46 smithi099 ceph-mon[109132]: from='client.14535 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:46.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:46 smithi135 ceph-mon[109341]: from='client.14535 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:46.703 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:46 smithi031 ceph-mon[107152]: from='client.14535 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:47.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:47 smithi031 ceph-mon[107152]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:47.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:47 smithi099 ceph-mon[109132]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:47.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:47 smithi135 ceph-mon[109341]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:47.808 INFO:teuthology.orchestra.run.smithi031.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T14:58:47.808 INFO:teuthology.orchestra.run.smithi031.stdout:mgr.x smithi031 *:9283,8765 running (4m) - 4m 489M - 18.0.0-5817-gbcea8c48 fa922c444906 44704039a3c5 2023-10-01T14:58:47.808 INFO:teuthology.orchestra.run.smithi031.stdout:mon.a smithi031 running (4m) - 4m 49.0M 2048M 18.0.0-5817-gbcea8c48 fa922c444906 d2a8ce6ea8e5 2023-10-01T14:58:47.808 INFO:teuthology.orchestra.run.smithi031.stdout:mon.b smithi099 running (3m) - 3m - 2048M 18.0.0-5817-gbcea8c48 fa922c444906 b3109440a90d 2023-10-01T14:58:47.808 INFO:teuthology.orchestra.run.smithi031.stdout:mon.c smithi135 running (3m) - 3m - 2048M 18.0.0-5817-gbcea8c48 fa922c444906 04dde62a2fbc 2023-10-01T14:58:47.808 INFO:teuthology.orchestra.run.smithi031.stdout:osd.0 smithi031 running (2m) - 2m 28.5M 8559M 18.0.0-5817-gbcea8c48 fa922c444906 552fe0a6c5a9 2023-10-01T14:58:47.808 INFO:teuthology.orchestra.run.smithi031.stdout:osd.1 smithi031 running (2m) - 2m 12.5M 8559M 18.0.0-5817-gbcea8c48 fa922c444906 1db4003c979e 2023-10-01T14:58:47.808 INFO:teuthology.orchestra.run.smithi031.stdout:osd.2 smithi099 running (2m) - 2m - 7069M 18.0.0-5817-gbcea8c48 fa922c444906 c17a744a607c 2023-10-01T14:58:47.808 INFO:teuthology.orchestra.run.smithi031.stdout:osd.3 smithi099 running (113s) - 113s - 7069M 18.0.0-5817-gbcea8c48 fa922c444906 251b1fb116c7 2023-10-01T14:58:47.808 INFO:teuthology.orchestra.run.smithi031.stdout:osd.4 smithi099 running (97s) - 97s - 7069M 18.0.0-5817-gbcea8c48 fa922c444906 fb0f82e04398 2023-10-01T14:58:47.809 INFO:teuthology.orchestra.run.smithi031.stdout:osd.5 smithi135 running (81s) - 81s - 7069M 18.0.0-5817-gbcea8c48 fa922c444906 ab7fc49fd82c 2023-10-01T14:58:47.809 INFO:teuthology.orchestra.run.smithi031.stdout:osd.6 smithi135 running (65s) - 65s - 7069M 18.0.0-5817-gbcea8c48 fa922c444906 4da5900c5444 2023-10-01T14:58:47.809 INFO:teuthology.orchestra.run.smithi031.stdout:osd.7 smithi135 running (49s) - 49s - 7069M 18.0.0-5817-gbcea8c48 fa922c444906 5d4f41d14ece 2023-10-01T14:58:48.200 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch ls' 2023-10-01T14:58:48.448 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:48.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:48 smithi031 ceph-mon[107152]: from='client.14541 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:48.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:48 smithi099 ceph-mon[109132]: from='client.14541 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:48.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:48 smithi135 ceph-mon[109341]: from='client.14541 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:49.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:49 smithi099 ceph-mon[109132]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:49.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:49 smithi135 ceph-mon[109341]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:49.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:49 smithi031 ceph-mon[107152]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:49.747 INFO:teuthology.orchestra.run.smithi031.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2023-10-01T14:58:49.747 INFO:teuthology.orchestra.run.smithi031.stdout:mgr 1/1 - 3m smithi031=x;count:1 2023-10-01T14:58:49.748 INFO:teuthology.orchestra.run.smithi031.stdout:mon 3/3 - 3m smithi031:172.21.15.31=a;smithi099:172.21.15.99=b;smithi135:172.21.15.135=c;count:3 2023-10-01T14:58:49.748 INFO:teuthology.orchestra.run.smithi031.stdout:osd 8 - - 2023-10-01T14:58:50.151 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch host ls' 2023-10-01T14:58:50.400 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:50.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:50 smithi031 ceph-mon[107152]: from='client.14547 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:50.588 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:50 smithi099 ceph-mon[109132]: from='client.14547 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:50.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:50 smithi135 ceph-mon[109341]: from='client.14547 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:51.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:51 smithi031 ceph-mon[107152]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:51.588 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:51 smithi099 ceph-mon[109132]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:51.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:51 smithi135 ceph-mon[109341]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:51.676 INFO:teuthology.orchestra.run.smithi031.stdout:HOST ADDR LABELS STATUS 2023-10-01T14:58:51.676 INFO:teuthology.orchestra.run.smithi031.stdout:smithi031 172.21.15.31 2023-10-01T14:58:51.676 INFO:teuthology.orchestra.run.smithi031.stdout:smithi099 172.21.15.99 2023-10-01T14:58:51.676 INFO:teuthology.orchestra.run.smithi031.stdout:smithi135 172.21.15.135 2023-10-01T14:58:51.676 INFO:teuthology.orchestra.run.smithi031.stdout:3 hosts in cluster 2023-10-01T14:58:52.114 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch device ls' 2023-10-01T14:58:52.362 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:52.587 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:52 smithi031 ceph-mon[107152]: from='client.14553 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:52.588 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:52 smithi099 ceph-mon[109132]: from='client.14553 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:52.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:52 smithi135 ceph-mon[109341]: from='client.14553 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:53.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:53 smithi099 ceph-mon[109132]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:53.628 INFO:teuthology.orchestra.run.smithi031.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REFRESHED REJECT REASONS 2023-10-01T14:58:53.628 INFO:teuthology.orchestra.run.smithi031.stdout:smithi031 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT53310008400BGN 372G No 2m ago LVM detected, locked 2023-10-01T14:58:53.628 INFO:teuthology.orchestra.run.smithi031.stdout:smithi099 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT62040101400BGN 372G No 99s ago LVM detected, locked 2023-10-01T14:58:53.628 INFO:teuthology.orchestra.run.smithi031.stdout:smithi135 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620401BA400BGN 372G No 51s ago LVM detected, locked 2023-10-01T14:58:53.652 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:53 smithi031 ceph-mon[107152]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:53.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:53 smithi135 ceph-mon[109341]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:54.041 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph osd lspools' 2023-10-01T14:58:54.285 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:54.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:54 smithi099 ceph-mon[109132]: from='client.14559 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:54.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:54 smithi135 ceph-mon[109341]: from='client.14559 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:54.736 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:54 smithi031 ceph-mon[107152]: from='client.14559 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:58:55.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:55 smithi031 ceph-mon[107152]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:55.569 INFO:teuthology.orchestra.run.smithi031.stdout:1 .mgr 2023-10-01T14:58:55.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:55 smithi099 ceph-mon[109132]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:55.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:55 smithi135 ceph-mon[109341]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:55.982 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph osd pool create mypool' 2023-10-01T14:58:56.232 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:56.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:56 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/2058791136' entity='client.admin' cmd=[{"prefix": "osd lspools"}]: dispatch 2023-10-01T14:58:56.636 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:56 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/2058791136' entity='client.admin' cmd=[{"prefix": "osd lspools"}]: dispatch 2023-10-01T14:58:56.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:56 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/2058791136' entity='client.admin' cmd=[{"prefix": "osd lspools"}]: dispatch 2023-10-01T14:58:57.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:57 smithi099 ceph-mon[109132]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:57.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:57 smithi135 ceph-mon[109341]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:57.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:57 smithi031 ceph-mon[107152]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:58.344 INFO:teuthology.orchestra.run.smithi031.stderr:pool 'mypool' created 2023-10-01T14:58:58.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:58 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/1737854204' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "mypool"}]: dispatch 2023-10-01T14:58:58.594 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:58 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/1737854204' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "mypool"}]: dispatch 2023-10-01T14:58:58.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:58 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/1737854204' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "mypool"}]: dispatch 2023-10-01T14:58:58.748 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph osd pool application enable mypool rbd' 2023-10-01T14:58:58.995 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:58:59.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:59 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/1737854204' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "mypool"}]': finished 2023-10-01T14:58:59.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:59 smithi031 ceph-mon[107152]: osdmap e50: 8 total, 8 up, 8 in 2023-10-01T14:58:59.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:58:59 smithi031 ceph-mon[107152]: pgmap v160: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:59.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:59 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/1737854204' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "mypool"}]': finished 2023-10-01T14:58:59.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:59 smithi135 ceph-mon[109341]: osdmap e50: 8 total, 8 up, 8 in 2023-10-01T14:58:59.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:58:59 smithi135 ceph-mon[109341]: pgmap v160: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:58:59.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:59 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/1737854204' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "mypool"}]': finished 2023-10-01T14:58:59.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:59 smithi099 ceph-mon[109132]: osdmap e50: 8 total, 8 up, 8 in 2023-10-01T14:58:59.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:58:59 smithi099 ceph-mon[109132]: pgmap v160: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:00.355 INFO:teuthology.orchestra.run.smithi031.stderr:enabled application 'rbd' on pool 'mypool' 2023-10-01T14:59:00.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:00 smithi135 ceph-mon[109341]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-01T14:59:00.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:00 smithi135 ceph-mon[109341]: osdmap e51: 8 total, 8 up, 8 in 2023-10-01T14:59:00.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:00 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/874028043' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "mypool", "app": "rbd"}]: dispatch 2023-10-01T14:59:00.673 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:00 smithi031 ceph-mon[107152]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-01T14:59:00.673 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:00 smithi031 ceph-mon[107152]: osdmap e51: 8 total, 8 up, 8 in 2023-10-01T14:59:00.673 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:00 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/874028043' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "mypool", "app": "rbd"}]: dispatch 2023-10-01T14:59:00.740 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph osd lspools' 2023-10-01T14:59:00.838 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:00 smithi099 ceph-mon[109132]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-01T14:59:00.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:00 smithi099 ceph-mon[109132]: osdmap e51: 8 total, 8 up, 8 in 2023-10-01T14:59:00.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:00 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/874028043' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "mypool", "app": "rbd"}]: dispatch 2023-10-01T14:59:00.990 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:59:01.608 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:01 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/874028043' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "mypool", "app": "rbd"}]': finished 2023-10-01T14:59:01.608 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:01 smithi031 ceph-mon[107152]: osdmap e52: 8 total, 8 up, 8 in 2023-10-01T14:59:01.608 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:01 smithi031 ceph-mon[107152]: pgmap v163: 33 pgs: 3 creating+peering, 21 active+clean, 9 unknown; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:01.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:01 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/874028043' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "mypool", "app": "rbd"}]': finished 2023-10-01T14:59:01.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:01 smithi135 ceph-mon[109341]: osdmap e52: 8 total, 8 up, 8 in 2023-10-01T14:59:01.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:01 smithi135 ceph-mon[109341]: pgmap v163: 33 pgs: 3 creating+peering, 21 active+clean, 9 unknown; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:01.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:01 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/874028043' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "mypool", "app": "rbd"}]': finished 2023-10-01T14:59:01.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:01 smithi099 ceph-mon[109132]: osdmap e52: 8 total, 8 up, 8 in 2023-10-01T14:59:01.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:01 smithi099 ceph-mon[109132]: pgmap v163: 33 pgs: 3 creating+peering, 21 active+clean, 9 unknown; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:02.232 INFO:teuthology.orchestra.run.smithi031.stdout:1 .mgr 2023-10-01T14:59:02.232 INFO:teuthology.orchestra.run.smithi031.stdout:2 mypool 2023-10-01T14:59:02.527 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:02 smithi031 ceph-mon[107152]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-01T14:59:02.527 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:02 smithi031 ceph-mon[107152]: Cluster is now healthy 2023-10-01T14:59:02.527 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:02 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/3177639627' entity='client.admin' cmd=[{"prefix": "osd lspools"}]: dispatch 2023-10-01T14:59:02.638 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph osd pool ls detail' 2023-10-01T14:59:02.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:02 smithi135 ceph-mon[109341]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-01T14:59:02.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:02 smithi135 ceph-mon[109341]: Cluster is now healthy 2023-10-01T14:59:02.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:02 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/3177639627' entity='client.admin' cmd=[{"prefix": "osd lspools"}]: dispatch 2023-10-01T14:59:02.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:02 smithi099 ceph-mon[109132]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-01T14:59:02.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:02 smithi099 ceph-mon[109132]: Cluster is now healthy 2023-10-01T14:59:02.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:02 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/3177639627' entity='client.admin' cmd=[{"prefix": "osd lspools"}]: dispatch 2023-10-01T14:59:02.896 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:59:03.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:03 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:03.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:03 smithi135 ceph-mon[109341]: pgmap v164: 33 pgs: 3 creating+peering, 30 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:03.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:03 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:03.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:03 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:03.670 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:03 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:03.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:03 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:03.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:03 smithi031 ceph-mon[107152]: pgmap v164: 33 pgs: 3 creating+peering, 30 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:03.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:03 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:03.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:03 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:03.745 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:03 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:03.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:03 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:03.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:03 smithi099 ceph-mon[109132]: pgmap v164: 33 pgs: 3 creating+peering, 30 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:03.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:03 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:03.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:03 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:03.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:03 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:04.221 INFO:teuthology.orchestra.run.smithi031.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-10-01T14:59:04.221 INFO:teuthology.orchestra.run.smithi031.stdout:pool 2 'mypool' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 52 flags hashpspool stripe_width 0 application rbd read_balance_score 1.75 2023-10-01T14:59:04.221 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:59:04.527 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:04 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/639253572' entity='client.admin' cmd=[{"prefix": "osd pool ls", "detail": "detail"}]: dispatch 2023-10-01T14:59:04.622 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mgr mgr/cephadm/container_image_nvmeof quay.io/barakda1/nvmeof:0.0.3' 2023-10-01T14:59:04.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:04 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/639253572' entity='client.admin' cmd=[{"prefix": "osd pool ls", "detail": "detail"}]: dispatch 2023-10-01T14:59:04.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:04 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/639253572' entity='client.admin' cmd=[{"prefix": "osd pool ls", "detail": "detail"}]: dispatch 2023-10-01T14:59:04.869 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:59:05.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:05 smithi135 ceph-mon[109341]: pgmap v165: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:05.696 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:05 smithi031 ceph-mon[107152]: pgmap v165: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:05.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:05 smithi099 ceph-mon[109132]: pgmap v165: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:06.555 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config get mgr mgr/cephadm/container_image_nvmeof' 2023-10-01T14:59:06.802 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:59:07.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:07 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/1005455326' entity='client.admin' 2023-10-01T14:59:07.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:07.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:07.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:07.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:07 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:07.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:07 smithi031 ceph-mon[107152]: pgmap v166: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:07.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:07 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/1005455326' entity='client.admin' 2023-10-01T14:59:07.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:07.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:07.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:07.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:07 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:07.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:07 smithi135 ceph-mon[109341]: pgmap v166: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:07.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:07 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/1005455326' entity='client.admin' 2023-10-01T14:59:07.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:07.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:07.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:07.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:07 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:07.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:07 smithi099 ceph-mon[109132]: pgmap v166: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:07.996 INFO:teuthology.orchestra.run.smithi031.stdout:quay.io/barakda1/nvmeof:0.0.3 2023-10-01T14:59:08.415 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:08 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/2433086955' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/container_image_nvmeof"}]: dispatch 2023-10-01T14:59:08.439 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch apply nvmeof mypool --placement="1 $(hostname)"' 2023-10-01T14:59:08.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:08 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/2433086955' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/container_image_nvmeof"}]: dispatch 2023-10-01T14:59:08.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:08 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/2433086955' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/container_image_nvmeof"}]: dispatch 2023-10-01T14:59:08.683 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:59:09.529 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:09 smithi031 ceph-mon[107152]: pgmap v167: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:09.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:09 smithi099 ceph-mon[109132]: pgmap v167: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:09.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:09 smithi135 ceph-mon[109341]: pgmap v167: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:09.991 INFO:teuthology.orchestra.run.smithi031.stdout:Scheduled nvmeof.mypool update... 2023-10-01T14:59:10.449 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch ps --refresh' 2023-10-01T14:59:10.720 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:59:11.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:10 smithi031 ceph-mon[107152]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "orch apply nvmeof", "pool": "mypool", "placement": "1 smithi031", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:59:11.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:10 smithi031 ceph-mon[107152]: Saving service nvmeof.mypool spec with placement smithi031;count:1 2023-10-01T14:59:11.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:11.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:11.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:11.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:11.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:11.246 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.nvmeof.mypool.smithi031.svnyug", "caps": ["mon", "profile rbd", "osd", "allow all tag rbd *=*"]}]: dispatch 2023-10-01T14:59:11.247 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "client.nvmeof.mypool.smithi031.svnyug", "caps": ["mon", "profile rbd", "osd", "allow all tag rbd *=*"]}]': finished 2023-10-01T14:59:11.247 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:10 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:11.247 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:10 smithi031 ceph-mon[107152]: Deploying daemon nvmeof.mypool.smithi031.svnyug on smithi031 2023-10-01T14:59:11.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:10 smithi099 ceph-mon[109132]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "orch apply nvmeof", "pool": "mypool", "placement": "1 smithi031", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:59:11.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:10 smithi099 ceph-mon[109132]: Saving service nvmeof.mypool spec with placement smithi031;count:1 2023-10-01T14:59:11.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:11.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:11.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:11.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:11.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:11.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.nvmeof.mypool.smithi031.svnyug", "caps": ["mon", "profile rbd", "osd", "allow all tag rbd *=*"]}]: dispatch 2023-10-01T14:59:11.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "client.nvmeof.mypool.smithi031.svnyug", "caps": ["mon", "profile rbd", "osd", "allow all tag rbd *=*"]}]': finished 2023-10-01T14:59:11.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:10 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:11.340 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:10 smithi099 ceph-mon[109132]: Deploying daemon nvmeof.mypool.smithi031.svnyug on smithi031 2023-10-01T14:59:11.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:10 smithi135 ceph-mon[109341]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "orch apply nvmeof", "pool": "mypool", "placement": "1 smithi031", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:59:11.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:10 smithi135 ceph-mon[109341]: Saving service nvmeof.mypool spec with placement smithi031;count:1 2023-10-01T14:59:11.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:11.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:11.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:11.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:11.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:11.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.nvmeof.mypool.smithi031.svnyug", "caps": ["mon", "profile rbd", "osd", "allow all tag rbd *=*"]}]: dispatch 2023-10-01T14:59:11.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "client.nvmeof.mypool.smithi031.svnyug", "caps": ["mon", "profile rbd", "osd", "allow all tag rbd *=*"]}]': finished 2023-10-01T14:59:11.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:10 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:11.420 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:10 smithi135 ceph-mon[109341]: Deploying daemon nvmeof.mypool.smithi031.svnyug on smithi031 2023-10-01T14:59:12.159 INFO:teuthology.orchestra.run.smithi031.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T14:59:12.160 INFO:teuthology.orchestra.run.smithi031.stdout:mgr.x smithi031 *:9283,8765 running (4m) - 4m 489M - 18.0.0-5817-gbcea8c48 fa922c444906 44704039a3c5 2023-10-01T14:59:12.160 INFO:teuthology.orchestra.run.smithi031.stdout:mon.a smithi031 running (4m) - 5m 49.0M 2048M 18.0.0-5817-gbcea8c48 fa922c444906 d2a8ce6ea8e5 2023-10-01T14:59:12.160 INFO:teuthology.orchestra.run.smithi031.stdout:mon.b smithi099 running (3m) - 3m - 2048M 18.0.0-5817-gbcea8c48 fa922c444906 b3109440a90d 2023-10-01T14:59:12.160 INFO:teuthology.orchestra.run.smithi031.stdout:mon.c smithi135 running (3m) - 3m - 2048M 18.0.0-5817-gbcea8c48 fa922c444906 04dde62a2fbc 2023-10-01T14:59:12.160 INFO:teuthology.orchestra.run.smithi031.stdout:osd.0 smithi031 running (3m) - 3m 28.5M 8559M 18.0.0-5817-gbcea8c48 fa922c444906 552fe0a6c5a9 2023-10-01T14:59:12.160 INFO:teuthology.orchestra.run.smithi031.stdout:osd.1 smithi031 running (2m) - 2m 12.5M 8559M 18.0.0-5817-gbcea8c48 fa922c444906 1db4003c979e 2023-10-01T14:59:12.160 INFO:teuthology.orchestra.run.smithi031.stdout:osd.2 smithi099 running (2m) - 2m - 7069M 18.0.0-5817-gbcea8c48 fa922c444906 c17a744a607c 2023-10-01T14:59:12.160 INFO:teuthology.orchestra.run.smithi031.stdout:osd.3 smithi099 running (2m) - 2m - 7069M 18.0.0-5817-gbcea8c48 fa922c444906 251b1fb116c7 2023-10-01T14:59:12.160 INFO:teuthology.orchestra.run.smithi031.stdout:osd.4 smithi099 running (2m) - 2m - 7069M 18.0.0-5817-gbcea8c48 fa922c444906 fb0f82e04398 2023-10-01T14:59:12.160 INFO:teuthology.orchestra.run.smithi031.stdout:osd.5 smithi135 running (106s) - 106s - 7069M 18.0.0-5817-gbcea8c48 fa922c444906 ab7fc49fd82c 2023-10-01T14:59:12.161 INFO:teuthology.orchestra.run.smithi031.stdout:osd.6 smithi135 running (90s) - 89s - 7069M 18.0.0-5817-gbcea8c48 fa922c444906 4da5900c5444 2023-10-01T14:59:12.161 INFO:teuthology.orchestra.run.smithi031.stdout:osd.7 smithi135 running (73s) - 73s - 7069M 18.0.0-5817-gbcea8c48 fa922c444906 5d4f41d14ece 2023-10-01T14:59:12.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:12 smithi099 ceph-mon[109132]: pgmap v168: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:12.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:12 smithi135 ceph-mon[109341]: pgmap v168: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:12.495 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:12 smithi031 ceph-mon[107152]: pgmap v168: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:12.622 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph versions' 2023-10-01T14:59:12.911 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:59:14.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:14 smithi031 ceph-mon[107152]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:59:14.245 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:14 smithi031 ceph-mon[107152]: pgmap v169: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:14.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:14 smithi099 ceph-mon[109132]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:59:14.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:14 smithi099 ceph-mon[109132]: pgmap v169: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:14.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:14 smithi135 ceph-mon[109341]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:59:14.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:14 smithi135 ceph-mon[109341]: pgmap v169: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:14.517 INFO:teuthology.orchestra.run.smithi031.stdout:{ 2023-10-01T14:59:14.517 INFO:teuthology.orchestra.run.smithi031.stdout: "mon": { 2023-10-01T14:59:14.517 INFO:teuthology.orchestra.run.smithi031.stdout: "ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev)": 3 2023-10-01T14:59:14.517 INFO:teuthology.orchestra.run.smithi031.stdout: }, 2023-10-01T14:59:14.517 INFO:teuthology.orchestra.run.smithi031.stdout: "mgr": { 2023-10-01T14:59:14.517 INFO:teuthology.orchestra.run.smithi031.stdout: "ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev)": 1 2023-10-01T14:59:14.517 INFO:teuthology.orchestra.run.smithi031.stdout: }, 2023-10-01T14:59:14.517 INFO:teuthology.orchestra.run.smithi031.stdout: "osd": { 2023-10-01T14:59:14.517 INFO:teuthology.orchestra.run.smithi031.stdout: "ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev)": 8 2023-10-01T14:59:14.518 INFO:teuthology.orchestra.run.smithi031.stdout: }, 2023-10-01T14:59:14.518 INFO:teuthology.orchestra.run.smithi031.stdout: "overall": { 2023-10-01T14:59:14.518 INFO:teuthology.orchestra.run.smithi031.stdout: "ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev)": 12 2023-10-01T14:59:14.518 INFO:teuthology.orchestra.run.smithi031.stdout: } 2023-10-01T14:59:14.518 INFO:teuthology.orchestra.run.smithi031.stdout:} 2023-10-01T14:59:15.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:15 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/4149891527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T14:59:15.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:15 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/4149891527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T14:59:15.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:15 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/4149891527' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T14:59:16.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:16 smithi099 ceph-mon[109132]: pgmap v170: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:16.419 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:16 smithi135 ceph-mon[109341]: pgmap v170: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:16.494 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:16 smithi031 ceph-mon[107152]: pgmap v170: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:17.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:17 smithi099 ceph-mon[109132]: pgmap v171: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:17.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:17 smithi135 ceph-mon[109341]: pgmap v171: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:17.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:17 smithi031 ceph-mon[107152]: pgmap v171: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:19.325 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph -s' 2023-10-01T14:59:19.868 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:59:19.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:19 smithi031 ceph-mon[107152]: pgmap v172: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:20.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:19 smithi099 ceph-mon[109132]: pgmap v172: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:20.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:19 smithi135 ceph-mon[109341]: pgmap v172: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:21.670 INFO:teuthology.orchestra.run.smithi031.stdout: cluster: 2023-10-01T14:59:21.671 INFO:teuthology.orchestra.run.smithi031.stdout: id: 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T14:59:21.671 INFO:teuthology.orchestra.run.smithi031.stdout: health: HEALTH_OK 2023-10-01T14:59:21.671 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:59:21.671 INFO:teuthology.orchestra.run.smithi031.stdout: services: 2023-10-01T14:59:21.671 INFO:teuthology.orchestra.run.smithi031.stdout: mon: 3 daemons, quorum a,c,b (age 3m) 2023-10-01T14:59:21.671 INFO:teuthology.orchestra.run.smithi031.stdout: mgr: x(active, since 4m) 2023-10-01T14:59:21.671 INFO:teuthology.orchestra.run.smithi031.stdout: osd: 8 osds: 8 up (since 77s), 8 in (since 91s) 2023-10-01T14:59:21.671 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:59:21.672 INFO:teuthology.orchestra.run.smithi031.stdout: data: 2023-10-01T14:59:21.672 INFO:teuthology.orchestra.run.smithi031.stdout: pools: 2 pools, 33 pgs 2023-10-01T14:59:21.672 INFO:teuthology.orchestra.run.smithi031.stdout: objects: 2 objects, 577 KiB 2023-10-01T14:59:21.672 INFO:teuthology.orchestra.run.smithi031.stdout: usage: 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:21.672 INFO:teuthology.orchestra.run.smithi031.stdout: pgs: 33 active+clean 2023-10-01T14:59:21.672 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:59:21.672 INFO:teuthology.orchestra.run.smithi031.stdout: progress: 2023-10-01T14:59:21.672 INFO:teuthology.orchestra.run.smithi031.stdout: Updating nvmeof.mypool deployment (+1 -> 1) (0s) 2023-10-01T14:59:21.672 INFO:teuthology.orchestra.run.smithi031.stdout: [............................] 2023-10-01T14:59:21.673 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:59:21.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:21 smithi099 ceph-mon[109132]: pgmap v173: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:21.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:21 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:21 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:21 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:21 smithi099 ceph-mon[109132]: Checking pool "mypool" exists for service nvmeof.mypool 2023-10-01T14:59:21.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:21 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:21 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:21.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:21 smithi099 ceph-mon[109132]: from='client.? 172.21.15.31:0/1546299402' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-01T14:59:21.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:21 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:21 smithi135 ceph-mon[109341]: pgmap v173: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:21.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:21 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:21 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:21 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:21 smithi135 ceph-mon[109341]: Checking pool "mypool" exists for service nvmeof.mypool 2023-10-01T14:59:21.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:21 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.919 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:21 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:21.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:21 smithi135 ceph-mon[109341]: from='client.? 172.21.15.31:0/1546299402' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-01T14:59:21.920 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:21 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:21 smithi031 ceph-mon[107152]: pgmap v173: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:21.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:21 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:21 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:21 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:21 smithi031 ceph-mon[107152]: Checking pool "mypool" exists for service nvmeof.mypool 2023-10-01T14:59:21.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:21 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:21.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:21 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:21.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:21 smithi031 ceph-mon[107152]: from='client.? 172.21.15.31:0/1546299402' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-01T14:59:21.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:21 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:22.095 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch ls' 2023-10-01T14:59:22.358 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:59:23.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:22 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:22 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:22 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:22 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:22 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:22 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:23.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:22 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:23.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:22 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.090 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:22 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:23.108 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:22 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.108 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:22 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.108 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:22 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.108 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:22 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.108 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:22 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.108 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:22 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:23.108 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:22 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:23.109 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:22 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.109 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:22 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:23.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:22 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:22 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:22 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:22 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:22 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:22 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:23.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:22 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:23.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:22 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:23.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:22 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T14:59:23.559 INFO:teuthology.orchestra.run.smithi031.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2023-10-01T14:59:23.559 INFO:teuthology.orchestra.run.smithi031.stdout:mgr 1/1 - 3m smithi031=x;count:1 2023-10-01T14:59:23.559 INFO:teuthology.orchestra.run.smithi031.stdout:mon 3/3 - 3m smithi031:172.21.15.31=a;smithi099:172.21.15.99=b;smithi135:172.21.15.135=c;count:3 2023-10-01T14:59:23.559 INFO:teuthology.orchestra.run.smithi031.stdout:nvmeof.mypool ?:4420,5500,8009 1/1 - 13s smithi031;count:1 2023-10-01T14:59:23.559 INFO:teuthology.orchestra.run.smithi031.stdout:osd 8 - - 2023-10-01T14:59:24.122 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:23 smithi031 ceph-mon[107152]: pgmap v174: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:24.122 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:23 smithi031 ceph-mon[107152]: mgrmap e15: x(active, since 4m) 2023-10-01T14:59:24.122 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:23 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:24.123 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'rbd create mypool/myimage --size 8Gi' 2023-10-01T14:59:24.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:23 smithi135 ceph-mon[109341]: pgmap v174: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:24.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:23 smithi135 ceph-mon[109341]: mgrmap e15: x(active, since 4m) 2023-10-01T14:59:24.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:23 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:24.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:23 smithi099 ceph-mon[109132]: pgmap v174: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-10-01T14:59:24.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:23 smithi099 ceph-mon[109132]: mgrmap e15: x(active, since 4m) 2023-10-01T14:59:24.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:23 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:24.383 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:59:25.133 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:24 smithi031 ceph-mon[107152]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:59:25.133 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:24 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:25.133 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:24 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:25.134 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:24 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:25.134 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:24 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:25.134 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:24 smithi031 ceph-mon[107152]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:25.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:24 smithi135 ceph-mon[109341]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:59:25.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:24 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:25.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:24 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:25.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:24 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:25.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:24 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:25.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:24 smithi135 ceph-mon[109341]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:25.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:24 smithi099 ceph-mon[109132]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T14:59:25.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:24 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:25.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:24 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:25.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:24 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T14:59:25.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:24 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T14:59:25.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:24 smithi099 ceph-mon[109132]: from='mgr.14152 172.21.15.31:0/1446076361' entity='mgr.x' 2023-10-01T14:59:26.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:25 smithi135 ceph-mon[109341]: pgmap v175: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2023-10-01T14:59:26.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:25 smithi031 ceph-mon[107152]: pgmap v175: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2023-10-01T14:59:26.339 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:25 smithi099 ceph-mon[109132]: pgmap v175: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2023-10-01T14:59:27.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:27 smithi031 ceph-mon[107152]: osdmap e53: 8 total, 8 up, 8 in 2023-10-01T14:59:27.514 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:27 smithi031 ceph-mon[107152]: pgmap v177: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 0 op/s 2023-10-01T14:59:27.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:27 smithi099 ceph-mon[109132]: osdmap e53: 8 total, 8 up, 8 in 2023-10-01T14:59:27.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:27 smithi099 ceph-mon[109132]: pgmap v177: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 0 op/s 2023-10-01T14:59:27.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:27 smithi135 ceph-mon[109341]: osdmap e53: 8 total, 8 up, 8 in 2023-10-01T14:59:27.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:27 smithi135 ceph-mon[109341]: pgmap v177: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 0 op/s 2023-10-01T14:59:27.695 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'rbd ls mypool' 2023-10-01T14:59:27.943 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:59:28.451 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:28 smithi031 ceph-mon[107152]: osdmap e54: 8 total, 8 up, 8 in 2023-10-01T14:59:28.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:28 smithi099 ceph-mon[109132]: osdmap e54: 8 total, 8 up, 8 in 2023-10-01T14:59:28.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:28 smithi135 ceph-mon[109341]: osdmap e54: 8 total, 8 up, 8 in 2023-10-01T14:59:28.840 INFO:teuthology.orchestra.run.smithi031.stdout:myimage 2023-10-01T14:59:29.280 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 -- bash -c 'set -ex 2023-10-01T14:59:29.280 DEBUG:teuthology.orchestra.run.smithi031:> # 2023-10-01T14:59:29.280 DEBUG:teuthology.orchestra.run.smithi031:> cat /proc/cpuinfo 2023-10-01T14:59:29.280 DEBUG:teuthology.orchestra.run.smithi031:> ' 2023-10-01T14:59:29.524 INFO:teuthology.orchestra.run.smithi031.stderr:Inferring config /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T14:59:29.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:29 smithi099 ceph-mon[109132]: pgmap v179: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 127 B/s wr, 0 op/s 2023-10-01T14:59:29.647 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:29 smithi031 ceph-mon[107152]: pgmap v179: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 127 B/s wr, 0 op/s 2023-10-01T14:59:29.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:29 smithi135 ceph-mon[109341]: pgmap v179: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 127 B/s wr, 0 op/s 2023-10-01T14:59:30.493 INFO:teuthology.orchestra.run.smithi031.stderr:+ cat /proc/cpuinfo 2023-10-01T14:59:30.494 INFO:teuthology.orchestra.run.smithi031.stdout:processor : 0 2023-10-01T14:59:30.494 INFO:teuthology.orchestra.run.smithi031.stdout:vendor_id : GenuineIntel 2023-10-01T14:59:30.494 INFO:teuthology.orchestra.run.smithi031.stdout:cpu family : 6 2023-10-01T14:59:30.494 INFO:teuthology.orchestra.run.smithi031.stdout:model : 63 2023-10-01T14:59:30.494 INFO:teuthology.orchestra.run.smithi031.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz 2023-10-01T14:59:30.494 INFO:teuthology.orchestra.run.smithi031.stdout:stepping : 2 2023-10-01T14:59:30.494 INFO:teuthology.orchestra.run.smithi031.stdout:microcode : 0x49 2023-10-01T14:59:30.494 INFO:teuthology.orchestra.run.smithi031.stdout:cpu MHz : 3500.119 2023-10-01T14:59:30.494 INFO:teuthology.orchestra.run.smithi031.stdout:cache size : 10240 KB 2023-10-01T14:59:30.494 INFO:teuthology.orchestra.run.smithi031.stdout:physical id : 0 2023-10-01T14:59:30.494 INFO:teuthology.orchestra.run.smithi031.stdout:siblings : 8 2023-10-01T14:59:30.495 INFO:teuthology.orchestra.run.smithi031.stdout:core id : 0 2023-10-01T14:59:30.495 INFO:teuthology.orchestra.run.smithi031.stdout:cpu cores : 4 2023-10-01T14:59:30.495 INFO:teuthology.orchestra.run.smithi031.stdout:apicid : 0 2023-10-01T14:59:30.495 INFO:teuthology.orchestra.run.smithi031.stdout:initial apicid : 0 2023-10-01T14:59:30.495 INFO:teuthology.orchestra.run.smithi031.stdout:fpu : yes 2023-10-01T14:59:30.495 INFO:teuthology.orchestra.run.smithi031.stdout:fpu_exception : yes 2023-10-01T14:59:30.495 INFO:teuthology.orchestra.run.smithi031.stdout:cpuid level : 15 2023-10-01T14:59:30.495 INFO:teuthology.orchestra.run.smithi031.stdout:wp : yes 2023-10-01T14:59:30.495 INFO:teuthology.orchestra.run.smithi031.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm cpuid_fault epb invpcid_single pti intel_ppin ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm xsaveopt cqm_llc cqm_occup_llc dtherm ida arat pln pts md_clear flush_l1d 2023-10-01T14:59:30.495 INFO:teuthology.orchestra.run.smithi031.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit mmio_stale_data 2023-10-01T14:59:30.495 INFO:teuthology.orchestra.run.smithi031.stdout:bogomips : 7000.23 2023-10-01T14:59:30.495 INFO:teuthology.orchestra.run.smithi031.stdout:clflush size : 64 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout:cache_alignment : 64 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout:power management: 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout:processor : 1 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout:vendor_id : GenuineIntel 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout:cpu family : 6 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout:model : 63 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout:stepping : 2 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout:microcode : 0x49 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout:cpu MHz : 3500.000 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout:cache size : 10240 KB 2023-10-01T14:59:30.496 INFO:teuthology.orchestra.run.smithi031.stdout:physical id : 0 2023-10-01T14:59:30.497 INFO:teuthology.orchestra.run.smithi031.stdout:siblings : 8 2023-10-01T14:59:30.497 INFO:teuthology.orchestra.run.smithi031.stdout:core id : 1 2023-10-01T14:59:30.497 INFO:teuthology.orchestra.run.smithi031.stdout:cpu cores : 4 2023-10-01T14:59:30.497 INFO:teuthology.orchestra.run.smithi031.stdout:apicid : 2 2023-10-01T14:59:30.497 INFO:teuthology.orchestra.run.smithi031.stdout:initial apicid : 2 2023-10-01T14:59:30.497 INFO:teuthology.orchestra.run.smithi031.stdout:fpu : yes 2023-10-01T14:59:30.497 INFO:teuthology.orchestra.run.smithi031.stdout:fpu_exception : yes 2023-10-01T14:59:30.497 INFO:teuthology.orchestra.run.smithi031.stdout:cpuid level : 15 2023-10-01T14:59:30.497 INFO:teuthology.orchestra.run.smithi031.stdout:wp : yes 2023-10-01T14:59:30.497 INFO:teuthology.orchestra.run.smithi031.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm cpuid_fault epb invpcid_single pti intel_ppin ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm xsaveopt cqm_llc cqm_occup_llc dtherm ida arat pln pts md_clear flush_l1d 2023-10-01T14:59:30.497 INFO:teuthology.orchestra.run.smithi031.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit mmio_stale_data 2023-10-01T14:59:30.497 INFO:teuthology.orchestra.run.smithi031.stdout:bogomips : 7000.23 2023-10-01T14:59:30.497 INFO:teuthology.orchestra.run.smithi031.stdout:clflush size : 64 2023-10-01T14:59:30.498 INFO:teuthology.orchestra.run.smithi031.stdout:cache_alignment : 64 2023-10-01T14:59:30.498 INFO:teuthology.orchestra.run.smithi031.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-01T14:59:30.498 INFO:teuthology.orchestra.run.smithi031.stdout:power management: 2023-10-01T14:59:30.498 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:59:30.498 INFO:teuthology.orchestra.run.smithi031.stdout:processor : 2 2023-10-01T14:59:30.498 INFO:teuthology.orchestra.run.smithi031.stdout:vendor_id : GenuineIntel 2023-10-01T14:59:30.498 INFO:teuthology.orchestra.run.smithi031.stdout:cpu family : 6 2023-10-01T14:59:30.498 INFO:teuthology.orchestra.run.smithi031.stdout:model : 63 2023-10-01T14:59:30.498 INFO:teuthology.orchestra.run.smithi031.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz 2023-10-01T14:59:30.498 INFO:teuthology.orchestra.run.smithi031.stdout:stepping : 2 2023-10-01T14:59:30.498 INFO:teuthology.orchestra.run.smithi031.stdout:microcode : 0x49 2023-10-01T14:59:30.498 INFO:teuthology.orchestra.run.smithi031.stdout:cpu MHz : 3500.149 2023-10-01T14:59:30.499 INFO:teuthology.orchestra.run.smithi031.stdout:cache size : 10240 KB 2023-10-01T14:59:30.499 INFO:teuthology.orchestra.run.smithi031.stdout:physical id : 0 2023-10-01T14:59:30.499 INFO:teuthology.orchestra.run.smithi031.stdout:siblings : 8 2023-10-01T14:59:30.499 INFO:teuthology.orchestra.run.smithi031.stdout:core id : 2 2023-10-01T14:59:30.499 INFO:teuthology.orchestra.run.smithi031.stdout:cpu cores : 4 2023-10-01T14:59:30.499 INFO:teuthology.orchestra.run.smithi031.stdout:apicid : 4 2023-10-01T14:59:30.499 INFO:teuthology.orchestra.run.smithi031.stdout:initial apicid : 4 2023-10-01T14:59:30.499 INFO:teuthology.orchestra.run.smithi031.stdout:fpu : yes 2023-10-01T14:59:30.499 INFO:teuthology.orchestra.run.smithi031.stdout:fpu_exception : yes 2023-10-01T14:59:30.499 INFO:teuthology.orchestra.run.smithi031.stdout:cpuid level : 15 2023-10-01T14:59:30.499 INFO:teuthology.orchestra.run.smithi031.stdout:wp : yes 2023-10-01T14:59:30.499 INFO:teuthology.orchestra.run.smithi031.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm cpuid_fault epb invpcid_single pti intel_ppin ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm xsaveopt cqm_llc cqm_occup_llc dtherm ida arat pln pts md_clear flush_l1d 2023-10-01T14:59:30.499 INFO:teuthology.orchestra.run.smithi031.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit mmio_stale_data 2023-10-01T14:59:30.500 INFO:teuthology.orchestra.run.smithi031.stdout:bogomips : 7000.23 2023-10-01T14:59:30.500 INFO:teuthology.orchestra.run.smithi031.stdout:clflush size : 64 2023-10-01T14:59:30.500 INFO:teuthology.orchestra.run.smithi031.stdout:cache_alignment : 64 2023-10-01T14:59:30.500 INFO:teuthology.orchestra.run.smithi031.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-01T14:59:30.500 INFO:teuthology.orchestra.run.smithi031.stdout:power management: 2023-10-01T14:59:30.500 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:59:30.500 INFO:teuthology.orchestra.run.smithi031.stdout:processor : 3 2023-10-01T14:59:30.500 INFO:teuthology.orchestra.run.smithi031.stdout:vendor_id : GenuineIntel 2023-10-01T14:59:30.500 INFO:teuthology.orchestra.run.smithi031.stdout:cpu family : 6 2023-10-01T14:59:30.500 INFO:teuthology.orchestra.run.smithi031.stdout:model : 63 2023-10-01T14:59:30.500 INFO:teuthology.orchestra.run.smithi031.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz 2023-10-01T14:59:30.500 INFO:teuthology.orchestra.run.smithi031.stdout:stepping : 2 2023-10-01T14:59:30.500 INFO:teuthology.orchestra.run.smithi031.stdout:microcode : 0x49 2023-10-01T14:59:30.501 INFO:teuthology.orchestra.run.smithi031.stdout:cpu MHz : 3477.435 2023-10-01T14:59:30.501 INFO:teuthology.orchestra.run.smithi031.stdout:cache size : 10240 KB 2023-10-01T14:59:30.501 INFO:teuthology.orchestra.run.smithi031.stdout:physical id : 0 2023-10-01T14:59:30.501 INFO:teuthology.orchestra.run.smithi031.stdout:siblings : 8 2023-10-01T14:59:30.501 INFO:teuthology.orchestra.run.smithi031.stdout:core id : 3 2023-10-01T14:59:30.501 INFO:teuthology.orchestra.run.smithi031.stdout:cpu cores : 4 2023-10-01T14:59:30.501 INFO:teuthology.orchestra.run.smithi031.stdout:apicid : 6 2023-10-01T14:59:30.501 INFO:teuthology.orchestra.run.smithi031.stdout:initial apicid : 6 2023-10-01T14:59:30.501 INFO:teuthology.orchestra.run.smithi031.stdout:fpu : yes 2023-10-01T14:59:30.501 INFO:teuthology.orchestra.run.smithi031.stdout:fpu_exception : yes 2023-10-01T14:59:30.501 INFO:teuthology.orchestra.run.smithi031.stdout:cpuid level : 15 2023-10-01T14:59:30.501 INFO:teuthology.orchestra.run.smithi031.stdout:wp : yes 2023-10-01T14:59:30.501 INFO:teuthology.orchestra.run.smithi031.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm cpuid_fault epb invpcid_single pti intel_ppin ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm xsaveopt cqm_llc cqm_occup_llc dtherm ida arat pln pts md_clear flush_l1d 2023-10-01T14:59:30.502 INFO:teuthology.orchestra.run.smithi031.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit mmio_stale_data 2023-10-01T14:59:30.502 INFO:teuthology.orchestra.run.smithi031.stdout:bogomips : 7000.23 2023-10-01T14:59:30.502 INFO:teuthology.orchestra.run.smithi031.stdout:clflush size : 64 2023-10-01T14:59:30.502 INFO:teuthology.orchestra.run.smithi031.stdout:cache_alignment : 64 2023-10-01T14:59:30.502 INFO:teuthology.orchestra.run.smithi031.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-01T14:59:30.502 INFO:teuthology.orchestra.run.smithi031.stdout:power management: 2023-10-01T14:59:30.502 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:59:30.502 INFO:teuthology.orchestra.run.smithi031.stdout:processor : 4 2023-10-01T14:59:30.502 INFO:teuthology.orchestra.run.smithi031.stdout:vendor_id : GenuineIntel 2023-10-01T14:59:30.502 INFO:teuthology.orchestra.run.smithi031.stdout:cpu family : 6 2023-10-01T14:59:30.502 INFO:teuthology.orchestra.run.smithi031.stdout:model : 63 2023-10-01T14:59:30.502 INFO:teuthology.orchestra.run.smithi031.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz 2023-10-01T14:59:30.502 INFO:teuthology.orchestra.run.smithi031.stdout:stepping : 2 2023-10-01T14:59:30.503 INFO:teuthology.orchestra.run.smithi031.stdout:microcode : 0x49 2023-10-01T14:59:30.503 INFO:teuthology.orchestra.run.smithi031.stdout:cpu MHz : 3500.000 2023-10-01T14:59:30.503 INFO:teuthology.orchestra.run.smithi031.stdout:cache size : 10240 KB 2023-10-01T14:59:30.503 INFO:teuthology.orchestra.run.smithi031.stdout:physical id : 0 2023-10-01T14:59:30.503 INFO:teuthology.orchestra.run.smithi031.stdout:siblings : 8 2023-10-01T14:59:30.503 INFO:teuthology.orchestra.run.smithi031.stdout:core id : 0 2023-10-01T14:59:30.503 INFO:teuthology.orchestra.run.smithi031.stdout:cpu cores : 4 2023-10-01T14:59:30.503 INFO:teuthology.orchestra.run.smithi031.stdout:apicid : 1 2023-10-01T14:59:30.503 INFO:teuthology.orchestra.run.smithi031.stdout:initial apicid : 1 2023-10-01T14:59:30.503 INFO:teuthology.orchestra.run.smithi031.stdout:fpu : yes 2023-10-01T14:59:30.503 INFO:teuthology.orchestra.run.smithi031.stdout:fpu_exception : yes 2023-10-01T14:59:30.503 INFO:teuthology.orchestra.run.smithi031.stdout:cpuid level : 15 2023-10-01T14:59:30.503 INFO:teuthology.orchestra.run.smithi031.stdout:wp : yes 2023-10-01T14:59:30.504 INFO:teuthology.orchestra.run.smithi031.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm cpuid_fault epb invpcid_single pti intel_ppin ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm xsaveopt cqm_llc cqm_occup_llc dtherm ida arat pln pts md_clear flush_l1d 2023-10-01T14:59:30.504 INFO:teuthology.orchestra.run.smithi031.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit mmio_stale_data 2023-10-01T14:59:30.504 INFO:teuthology.orchestra.run.smithi031.stdout:bogomips : 7000.23 2023-10-01T14:59:30.504 INFO:teuthology.orchestra.run.smithi031.stdout:clflush size : 64 2023-10-01T14:59:30.504 INFO:teuthology.orchestra.run.smithi031.stdout:cache_alignment : 64 2023-10-01T14:59:30.504 INFO:teuthology.orchestra.run.smithi031.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-01T14:59:30.504 INFO:teuthology.orchestra.run.smithi031.stdout:power management: 2023-10-01T14:59:30.504 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:59:30.504 INFO:teuthology.orchestra.run.smithi031.stdout:processor : 5 2023-10-01T14:59:30.504 INFO:teuthology.orchestra.run.smithi031.stdout:vendor_id : GenuineIntel 2023-10-01T14:59:30.504 INFO:teuthology.orchestra.run.smithi031.stdout:cpu family : 6 2023-10-01T14:59:30.504 INFO:teuthology.orchestra.run.smithi031.stdout:model : 63 2023-10-01T14:59:30.505 INFO:teuthology.orchestra.run.smithi031.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz 2023-10-01T14:59:30.505 INFO:teuthology.orchestra.run.smithi031.stdout:stepping : 2 2023-10-01T14:59:30.505 INFO:teuthology.orchestra.run.smithi031.stdout:microcode : 0x49 2023-10-01T14:59:30.505 INFO:teuthology.orchestra.run.smithi031.stdout:cpu MHz : 3500.000 2023-10-01T14:59:30.505 INFO:teuthology.orchestra.run.smithi031.stdout:cache size : 10240 KB 2023-10-01T14:59:30.505 INFO:teuthology.orchestra.run.smithi031.stdout:physical id : 0 2023-10-01T14:59:30.505 INFO:teuthology.orchestra.run.smithi031.stdout:siblings : 8 2023-10-01T14:59:30.505 INFO:teuthology.orchestra.run.smithi031.stdout:core id : 1 2023-10-01T14:59:30.505 INFO:teuthology.orchestra.run.smithi031.stdout:cpu cores : 4 2023-10-01T14:59:30.505 INFO:teuthology.orchestra.run.smithi031.stdout:apicid : 3 2023-10-01T14:59:30.505 INFO:teuthology.orchestra.run.smithi031.stdout:initial apicid : 3 2023-10-01T14:59:30.505 INFO:teuthology.orchestra.run.smithi031.stdout:fpu : yes 2023-10-01T14:59:30.505 INFO:teuthology.orchestra.run.smithi031.stdout:fpu_exception : yes 2023-10-01T14:59:30.506 INFO:teuthology.orchestra.run.smithi031.stdout:cpuid level : 15 2023-10-01T14:59:30.506 INFO:teuthology.orchestra.run.smithi031.stdout:wp : yes 2023-10-01T14:59:30.506 INFO:teuthology.orchestra.run.smithi031.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm cpuid_fault epb invpcid_single pti intel_ppin ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm xsaveopt cqm_llc cqm_occup_llc dtherm ida arat pln pts md_clear flush_l1d 2023-10-01T14:59:30.506 INFO:teuthology.orchestra.run.smithi031.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit mmio_stale_data 2023-10-01T14:59:30.506 INFO:teuthology.orchestra.run.smithi031.stdout:bogomips : 7000.23 2023-10-01T14:59:30.506 INFO:teuthology.orchestra.run.smithi031.stdout:clflush size : 64 2023-10-01T14:59:30.506 INFO:teuthology.orchestra.run.smithi031.stdout:cache_alignment : 64 2023-10-01T14:59:30.506 INFO:teuthology.orchestra.run.smithi031.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-01T14:59:30.506 INFO:teuthology.orchestra.run.smithi031.stdout:power management: 2023-10-01T14:59:30.506 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:59:30.506 INFO:teuthology.orchestra.run.smithi031.stdout:processor : 6 2023-10-01T14:59:30.506 INFO:teuthology.orchestra.run.smithi031.stdout:vendor_id : GenuineIntel 2023-10-01T14:59:30.506 INFO:teuthology.orchestra.run.smithi031.stdout:cpu family : 6 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:model : 63 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:stepping : 2 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:microcode : 0x49 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:cpu MHz : 3500.000 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:cache size : 10240 KB 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:physical id : 0 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:siblings : 8 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:core id : 2 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:cpu cores : 4 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:apicid : 5 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:initial apicid : 5 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:fpu : yes 2023-10-01T14:59:30.507 INFO:teuthology.orchestra.run.smithi031.stdout:fpu_exception : yes 2023-10-01T14:59:30.508 INFO:teuthology.orchestra.run.smithi031.stdout:cpuid level : 15 2023-10-01T14:59:30.508 INFO:teuthology.orchestra.run.smithi031.stdout:wp : yes 2023-10-01T14:59:30.508 INFO:teuthology.orchestra.run.smithi031.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm cpuid_fault epb invpcid_single pti intel_ppin ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm xsaveopt cqm_llc cqm_occup_llc dtherm ida arat pln pts md_clear flush_l1d 2023-10-01T14:59:30.508 INFO:teuthology.orchestra.run.smithi031.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit mmio_stale_data 2023-10-01T14:59:30.508 INFO:teuthology.orchestra.run.smithi031.stdout:bogomips : 7000.23 2023-10-01T14:59:30.508 INFO:teuthology.orchestra.run.smithi031.stdout:clflush size : 64 2023-10-01T14:59:30.508 INFO:teuthology.orchestra.run.smithi031.stdout:cache_alignment : 64 2023-10-01T14:59:30.508 INFO:teuthology.orchestra.run.smithi031.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-01T14:59:30.508 INFO:teuthology.orchestra.run.smithi031.stdout:power management: 2023-10-01T14:59:30.508 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:59:30.508 INFO:teuthology.orchestra.run.smithi031.stdout:processor : 7 2023-10-01T14:59:30.508 INFO:teuthology.orchestra.run.smithi031.stdout:vendor_id : GenuineIntel 2023-10-01T14:59:30.508 INFO:teuthology.orchestra.run.smithi031.stdout:cpu family : 6 2023-10-01T14:59:30.509 INFO:teuthology.orchestra.run.smithi031.stdout:model : 63 2023-10-01T14:59:30.509 INFO:teuthology.orchestra.run.smithi031.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz 2023-10-01T14:59:30.509 INFO:teuthology.orchestra.run.smithi031.stdout:stepping : 2 2023-10-01T14:59:30.509 INFO:teuthology.orchestra.run.smithi031.stdout:microcode : 0x49 2023-10-01T14:59:30.509 INFO:teuthology.orchestra.run.smithi031.stdout:cpu MHz : 3500.141 2023-10-01T14:59:30.509 INFO:teuthology.orchestra.run.smithi031.stdout:cache size : 10240 KB 2023-10-01T14:59:30.509 INFO:teuthology.orchestra.run.smithi031.stdout:physical id : 0 2023-10-01T14:59:30.509 INFO:teuthology.orchestra.run.smithi031.stdout:siblings : 8 2023-10-01T14:59:30.509 INFO:teuthology.orchestra.run.smithi031.stdout:core id : 3 2023-10-01T14:59:30.509 INFO:teuthology.orchestra.run.smithi031.stdout:cpu cores : 4 2023-10-01T14:59:30.509 INFO:teuthology.orchestra.run.smithi031.stdout:apicid : 7 2023-10-01T14:59:30.509 INFO:teuthology.orchestra.run.smithi031.stdout:initial apicid : 7 2023-10-01T14:59:30.509 INFO:teuthology.orchestra.run.smithi031.stdout:fpu : yes 2023-10-01T14:59:30.510 INFO:teuthology.orchestra.run.smithi031.stdout:fpu_exception : yes 2023-10-01T14:59:30.510 INFO:teuthology.orchestra.run.smithi031.stdout:cpuid level : 15 2023-10-01T14:59:30.510 INFO:teuthology.orchestra.run.smithi031.stdout:wp : yes 2023-10-01T14:59:30.510 INFO:teuthology.orchestra.run.smithi031.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm cpuid_fault epb invpcid_single pti intel_ppin ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm xsaveopt cqm_llc cqm_occup_llc dtherm ida arat pln pts md_clear flush_l1d 2023-10-01T14:59:30.510 INFO:teuthology.orchestra.run.smithi031.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit mmio_stale_data 2023-10-01T14:59:30.510 INFO:teuthology.orchestra.run.smithi031.stdout:bogomips : 7000.23 2023-10-01T14:59:30.510 INFO:teuthology.orchestra.run.smithi031.stdout:clflush size : 64 2023-10-01T14:59:30.510 INFO:teuthology.orchestra.run.smithi031.stdout:cache_alignment : 64 2023-10-01T14:59:30.510 INFO:teuthology.orchestra.run.smithi031.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-01T14:59:30.510 INFO:teuthology.orchestra.run.smithi031.stdout:power management: 2023-10-01T14:59:30.510 INFO:teuthology.orchestra.run.smithi031.stdout: 2023-10-01T14:59:30.871 INFO:teuthology.run_tasks:Running task workunit... 2023-10-01T14:59:30.884 INFO:tasks.workunit:Pulling workunits from ref 71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5 2023-10-01T14:59:30.884 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-01T14:59:30.884 DEBUG:teuthology.orchestra.run.smithi031:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-01T14:59:30.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T14:59:30.904 INFO:teuthology.orchestra.run.smithi031.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-01T14:59:30.904 DEBUG:teuthology.orchestra.run.smithi031:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-01T14:59:30.964 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-01T14:59:30.964 DEBUG:teuthology.orchestra.run.smithi031:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-01T14:59:31.022 INFO:tasks.workunit:timeout=3h 2023-10-01T14:59:31.022 INFO:tasks.workunit:cleanup=True 2023-10-01T14:59:31.023 DEBUG:teuthology.orchestra.run.smithi031:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://github.com/barakda/ceph /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5 2023-10-01T14:59:31.083 INFO:tasks.workunit.client.0.smithi031.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-01T14:59:31.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:31 smithi031 ceph-mon[107152]: pgmap v180: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 895 B/s wr, 2 op/s 2023-10-01T14:59:32.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:31 smithi099 ceph-mon[109132]: pgmap v180: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 895 B/s wr, 2 op/s 2023-10-01T14:59:32.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:31 smithi135 ceph-mon[109341]: pgmap v180: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 895 B/s wr, 2 op/s 2023-10-01T14:59:33.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:33 smithi031 ceph-mon[107152]: pgmap v181: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 767 B/s wr, 1 op/s 2023-10-01T14:59:34.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:33 smithi099 ceph-mon[109132]: pgmap v181: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 767 B/s wr, 1 op/s 2023-10-01T14:59:34.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:33 smithi135 ceph-mon[109341]: pgmap v181: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 767 B/s wr, 1 op/s 2023-10-01T14:59:35.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:35 smithi031 ceph-mon[107152]: pgmap v182: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 731 B/s wr, 1 op/s 2023-10-01T14:59:36.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:35 smithi099 ceph-mon[109132]: pgmap v182: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 731 B/s wr, 1 op/s 2023-10-01T14:59:36.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:35 smithi135 ceph-mon[109341]: pgmap v182: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 731 B/s wr, 1 op/s 2023-10-01T14:59:37.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:37 smithi099 ceph-mon[109132]: osdmap e55: 8 total, 8 up, 8 in 2023-10-01T14:59:37.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:37 smithi099 ceph-mon[109132]: pgmap v184: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 653 B/s rd, 653 B/s wr, 1 op/s 2023-10-01T14:59:37.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:37 smithi135 ceph-mon[109341]: osdmap e55: 8 total, 8 up, 8 in 2023-10-01T14:59:37.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:37 smithi135 ceph-mon[109341]: pgmap v184: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 653 B/s rd, 653 B/s wr, 1 op/s 2023-10-01T14:59:37.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:37 smithi031 ceph-mon[107152]: osdmap e55: 8 total, 8 up, 8 in 2023-10-01T14:59:37.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:37 smithi031 ceph-mon[107152]: pgmap v184: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 653 B/s rd, 653 B/s wr, 1 op/s 2023-10-01T14:59:39.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:39 smithi031 ceph-mon[107152]: pgmap v185: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 614 B/s rd, 614 B/s wr, 1 op/s 2023-10-01T14:59:40.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:39 smithi099 ceph-mon[109132]: pgmap v185: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 614 B/s rd, 614 B/s wr, 1 op/s 2023-10-01T14:59:40.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:39 smithi135 ceph-mon[109341]: pgmap v185: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 614 B/s rd, 614 B/s wr, 1 op/s 2023-10-01T14:59:41.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:41 smithi031 ceph-mon[107152]: pgmap v186: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2023-10-01T14:59:42.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:41 smithi099 ceph-mon[109132]: pgmap v186: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2023-10-01T14:59:42.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:41 smithi135 ceph-mon[109341]: pgmap v186: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2023-10-01T14:59:43.995 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:43 smithi031 ceph-mon[107152]: pgmap v187: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2023-10-01T14:59:44.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:43 smithi099 ceph-mon[109132]: pgmap v187: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2023-10-01T14:59:44.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:43 smithi135 ceph-mon[109341]: pgmap v187: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2023-10-01T14:59:45.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:45 smithi031 ceph-mon[107152]: pgmap v188: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2023-10-01T14:59:46.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:45 smithi099 ceph-mon[109132]: pgmap v188: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2023-10-01T14:59:46.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:45 smithi135 ceph-mon[109341]: pgmap v188: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2023-10-01T14:59:47.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:47 smithi099 ceph-mon[109132]: pgmap v189: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 197 B/s rd, 0 op/s 2023-10-01T14:59:47.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:47 smithi135 ceph-mon[109341]: pgmap v189: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 197 B/s rd, 0 op/s 2023-10-01T14:59:47.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:47 smithi031 ceph-mon[107152]: pgmap v189: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 197 B/s rd, 0 op/s 2023-10-01T14:59:49.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:49 smithi031 ceph-mon[107152]: pgmap v190: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T14:59:50.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:49 smithi099 ceph-mon[109132]: pgmap v190: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T14:59:50.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:49 smithi135 ceph-mon[109341]: pgmap v190: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T14:59:51.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:51 smithi031 ceph-mon[107152]: pgmap v191: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-10-01T14:59:52.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:51 smithi099 ceph-mon[109132]: pgmap v191: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-10-01T14:59:52.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:51 smithi135 ceph-mon[109341]: pgmap v191: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-10-01T14:59:53.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:53 smithi031 ceph-mon[107152]: pgmap v192: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T14:59:54.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:53 smithi099 ceph-mon[109132]: pgmap v192: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T14:59:54.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:53 smithi135 ceph-mon[109341]: pgmap v192: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T14:59:55.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:55 smithi031 ceph-mon[107152]: pgmap v193: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-10-01T14:59:56.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:55 smithi099 ceph-mon[109132]: pgmap v193: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-10-01T14:59:56.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:55 smithi135 ceph-mon[109341]: pgmap v193: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-10-01T14:59:57.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:57 smithi099 ceph-mon[109132]: pgmap v194: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T14:59:57.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:57 smithi135 ceph-mon[109341]: pgmap v194: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T14:59:57.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:57 smithi031 ceph-mon[107152]: pgmap v194: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T14:59:59.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 14:59:59 smithi031 ceph-mon[107152]: pgmap v195: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T15:00:00.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 14:59:59 smithi099 ceph-mon[109132]: pgmap v195: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T15:00:00.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 14:59:59 smithi135 ceph-mon[109341]: pgmap v195: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T15:00:00.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 15:00:00 smithi031 ceph-mon[107152]: overall HEALTH_OK 2023-10-01T15:00:01.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 15:00:00 smithi099 ceph-mon[109132]: overall HEALTH_OK 2023-10-01T15:00:01.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 15:00:00 smithi135 ceph-mon[109341]: overall HEALTH_OK 2023-10-01T15:00:01.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 15:00:01 smithi031 ceph-mon[107152]: pgmap v196: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-10-01T15:00:02.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 15:00:01 smithi099 ceph-mon[109132]: pgmap v196: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-10-01T15:00:02.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 15:00:01 smithi135 ceph-mon[109341]: pgmap v196: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-10-01T15:00:03.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 15:00:03 smithi031 ceph-mon[107152]: pgmap v197: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T15:00:04.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 15:00:03 smithi099 ceph-mon[109132]: pgmap v197: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T15:00:04.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 15:00:03 smithi135 ceph-mon[109341]: pgmap v197: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T15:00:05.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 15:00:05 smithi031 ceph-mon[107152]: pgmap v198: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-10-01T15:00:06.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 15:00:05 smithi099 ceph-mon[109132]: pgmap v198: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-10-01T15:00:06.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 15:00:05 smithi135 ceph-mon[109341]: pgmap v198: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2023-10-01T15:00:07.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 15:00:07 smithi099 ceph-mon[109132]: pgmap v199: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T15:00:07.669 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 15:00:07 smithi135 ceph-mon[109341]: pgmap v199: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T15:00:07.744 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 15:00:07 smithi031 ceph-mon[107152]: pgmap v199: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T15:00:08.764 INFO:tasks.workunit.client.0.smithi031.stderr:Updating files: 95% (11548/12039) Updating files: 96% (11558/12039) Updating files: 97% (11678/12039) Updating files: 98% (11799/12039) Updating files: 99% (11919/12039) Updating files: 100% (12039/12039) Updating files: 100% (12039/12039), done. 2023-10-01T15:00:09.337 INFO:tasks.workunit.client.0.smithi031.stderr:Note: switching to '71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5'. 2023-10-01T15:00:09.466 INFO:tasks.workunit.client.0.smithi031.stderr: 2023-10-01T15:00:09.466 INFO:tasks.workunit.client.0.smithi031.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2023-10-01T15:00:09.466 INFO:tasks.workunit.client.0.smithi031.stderr:changes and commit them, and you can discard any commits you make in this 2023-10-01T15:00:09.466 INFO:tasks.workunit.client.0.smithi031.stderr:state without impacting any branches by switching back to a branch. 2023-10-01T15:00:09.466 INFO:tasks.workunit.client.0.smithi031.stderr: 2023-10-01T15:00:09.466 INFO:tasks.workunit.client.0.smithi031.stderr:If you want to create a new branch to retain commits you create, you may 2023-10-01T15:00:09.466 INFO:tasks.workunit.client.0.smithi031.stderr:do so (now or later) by using -c with the switch command. Example: 2023-10-01T15:00:09.466 INFO:tasks.workunit.client.0.smithi031.stderr: 2023-10-01T15:00:09.466 INFO:tasks.workunit.client.0.smithi031.stderr: git switch -c 2023-10-01T15:00:09.467 INFO:tasks.workunit.client.0.smithi031.stderr: 2023-10-01T15:00:09.467 INFO:tasks.workunit.client.0.smithi031.stderr:Or undo this operation with: 2023-10-01T15:00:09.467 INFO:tasks.workunit.client.0.smithi031.stderr: 2023-10-01T15:00:09.467 INFO:tasks.workunit.client.0.smithi031.stderr: git switch - 2023-10-01T15:00:09.467 INFO:tasks.workunit.client.0.smithi031.stderr: 2023-10-01T15:00:09.467 INFO:tasks.workunit.client.0.smithi031.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2023-10-01T15:00:09.467 INFO:tasks.workunit.client.0.smithi031.stderr: 2023-10-01T15:00:09.467 INFO:tasks.workunit.client.0.smithi031.stderr:HEAD is now at 71faca2bb8 nvme test 2023-10-01T15:00:09.468 DEBUG:teuthology.orchestra.run.smithi031:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-01T15:00:09.541 INFO:tasks.workunit.client.0.smithi031.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-01T15:00:09.543 INFO:tasks.workunit.client.0.smithi031.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-01T15:00:09.543 INFO:tasks.workunit.client.0.smithi031.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-01T15:00:09.994 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 15:00:09 smithi031 ceph-mon[107152]: pgmap v200: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T15:00:10.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 15:00:09 smithi099 ceph-mon[109132]: pgmap v200: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T15:00:10.102 INFO:tasks.workunit.client.0.smithi031.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-01T15:00:10.164 INFO:tasks.workunit.client.0.smithi031.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-01T15:00:10.169 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 15:00:09 smithi135 ceph-mon[109341]: pgmap v200: 33 pgs: 33 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2023-10-01T15:00:10.205 INFO:tasks.workunit.client.0.smithi031.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-01T15:00:10.207 INFO:tasks.workunit.client.0.smithi031.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-01T15:00:10.207 INFO:tasks.workunit.client.0.smithi031.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-01T15:00:10.247 INFO:tasks.workunit.client.0.smithi031.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-01T15:00:10.253 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2023-10-01T15:00:10.253 DEBUG:teuthology.orchestra.run.smithi031:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-01T15:00:10.314 INFO:tasks.workunit:Running workunits matching cephadm/test_nvme_etc_hosts.sh on client.0... 2023-10-01T15:00:10.315 INFO:tasks.workunit:Running workunit cephadm/test_nvme_etc_hosts.sh... 2023-10-01T15:00:10.315 DEBUG:teuthology.orchestra.run.smithi031:workunit test cephadm/test_nvme_etc_hosts.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_nvme_etc_hosts.sh 2023-10-01T15:00:10.373 DEBUG:teuthology.orchestra.run:got remote process result: 127 2023-10-01T15:00:10.373 INFO:tasks.workunit.client.0.smithi031.stderr:bash: adjust-ulimits: command not found 2023-10-01T15:00:10.374 INFO:tasks.workunit:Stopping ['cephadm/test_nvme_etc_hosts.sh', 'set -ex\n#\nHOSTNAME=$(hostname)\nIP=$(echo $(hostname -I) | cut -d \' \' -f1)\necho -e "<---- exec.client.0---- HOST/IP -- $HOSTNAME/$IP ---->"\n'] on client.0... 2023-10-01T15:00:10.374 DEBUG:teuthology.orchestra.run.smithi031:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-01T15:00:10.693 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_barakda_ceph_71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5/qa/tasks/workunit.py", line 129, in task p.spawn(_run_tests, ctx, refspec, role, tests, File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/github.com_barakda_ceph_71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5/qa/tasks/workunit.py", line 424, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/remote.py", line 522, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cephadm/test_nvme_etc_hosts.sh) on smithi031 with status 127: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_nvme_etc_hosts.sh' 2023-10-01T15:00:10.874 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=d852654015ed48f8a464f112d6212444 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_barakda_ceph_71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5/qa/tasks/workunit.py", line 129, in task p.spawn(_run_tests, ctx, refspec, role, tests, File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/github.com_barakda_ceph_71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5/qa/tasks/workunit.py", line 424, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/remote.py", line 522, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cephadm/test_nvme_etc_hosts.sh) on smithi031 with status 127: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_nvme_etc_hosts.sh' 2023-10-01T15:00:10.877 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-01T15:00:10.886 INFO:tasks.cephadm:Teardown begin 2023-10-01T15:00:10.887 DEBUG:teuthology.orchestra.run.smithi031:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T15:00:10.930 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T15:00:10.972 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T15:00:11.014 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-01T15:00:11.015 DEBUG:teuthology.orchestra.run.smithi031:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-01T15:00:11.034 DEBUG:teuthology.orchestra.run.smithi099:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-01T15:00:11.055 DEBUG:teuthology.orchestra.run.smithi135:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-01T15:00:11.075 INFO:tasks.cephadm:Stopping all daemons... 2023-10-01T15:00:11.075 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-01T15:00:11.075 DEBUG:teuthology.orchestra.run.smithi031:> sudo systemctl stop ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.a 2023-10-01T15:00:11.244 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 15:00:11 smithi031 systemd[1]: Stopping Ceph mon.a for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T15:00:11.580 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 15:00:11 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mon-a[107148]: 2023-10-01T15:00:11.308+0000 7f3c6cadc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T15:00:11.581 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 15:00:11 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mon-a[107148]: 2023-10-01T15:00:11.308+0000 7f3c6cadc700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-01T15:00:11.926 INFO:journalctl@ceph.mon.a.smithi031.stdout:Oct 01 15:00:11 smithi031 bash[134655]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mon-a 2023-10-01T15:00:11.935 DEBUG:teuthology.orchestra.run.smithi031:> sudo pkill -f 'journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-01T15:00:11.987 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T15:00:11.987 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-01T15:00:11.987 INFO:tasks.cephadm.mon.c:Stopping mon.b... 2023-10-01T15:00:11.987 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.b 2023-10-01T15:00:12.279 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 15:00:12 smithi099 systemd[1]: Stopping Ceph mon.b for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T15:00:12.279 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 15:00:12 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mon-b[109128]: 2023-10-01T15:00:12.196+0000 7fbc4c932700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T15:00:12.280 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 15:00:12 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mon-b[109128]: 2023-10-01T15:00:12.196+0000 7fbc4c932700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2023-10-01T15:00:12.588 INFO:journalctl@ceph.mon.b.smithi099.stdout:Oct 01 15:00:12 smithi099 bash[124259]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mon-b 2023-10-01T15:00:12.639 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-01T15:00:12.686 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T15:00:12.686 INFO:tasks.cephadm.mon.c:Stopped mon.b 2023-10-01T15:00:12.686 INFO:tasks.cephadm.mon.c:Stopping mon.c... 2023-10-01T15:00:12.687 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.c 2023-10-01T15:00:13.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 15:00:12 smithi135 systemd[1]: Stopping Ceph mon.c for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T15:00:13.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 15:00:12 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mon-c[109323]: 2023-10-01T15:00:12.914+0000 7fdecbc02700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T15:00:13.170 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 15:00:12 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mon-c[109323]: 2023-10-01T15:00:12.914+0000 7fdecbc02700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2023-10-01T15:00:13.538 INFO:journalctl@ceph.mon.c.smithi135.stdout:Oct 01 15:00:13 smithi135 bash[125187]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mon-c 2023-10-01T15:00:13.544 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-01T15:00:13.589 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T15:00:13.589 INFO:tasks.cephadm.mon.c:Stopped mon.c 2023-10-01T15:00:13.589 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-01T15:00:13.589 DEBUG:teuthology.orchestra.run.smithi031:> sudo systemctl stop ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mgr.x 2023-10-01T15:00:13.965 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 15:00:13 smithi031 systemd[1]: Stopping Ceph mgr.x for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T15:00:13.965 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 15:00:13 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x[107378]: 2023-10-01T15:00:13.814+0000 7fe3b11fc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.x -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T15:00:14.244 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Oct 01 15:00:14 smithi031 bash[134895]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7-mgr-x 2023-10-01T15:00:14.289 DEBUG:teuthology.orchestra.run.smithi031:> sudo pkill -f 'journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-01T15:00:14.329 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T15:00:14.329 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-01T15:00:14.330 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-01T15:00:14.330 DEBUG:teuthology.orchestra.run.smithi031:> sudo systemctl stop ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.0 2023-10-01T15:00:14.593 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 15:00:14 smithi031 systemd[1]: Stopping Ceph osd.0 for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T15:00:14.994 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 15:00:14 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-0[118577]: 2023-10-01T15:00:14.590+0000 7fce5cd17700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T15:00:14.995 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 15:00:14 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-0[118577]: 2023-10-01T15:00:14.590+0000 7fce5cd17700 -1 osd.0 55 *** Got signal Terminated *** 2023-10-01T15:00:14.995 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 15:00:14 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-0[118577]: 2023-10-01T15:00:14.590+0000 7fce5cd17700 -1 osd.0 55 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T15:00:20.494 INFO:journalctl@ceph.osd.0.smithi031.stdout:Oct 01 15:00:20 smithi031 bash[135030]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-0 2023-10-01T15:00:21.306 DEBUG:teuthology.orchestra.run.smithi031:> sudo pkill -f 'journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-01T15:00:21.350 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T15:00:21.350 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-01T15:00:21.351 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-01T15:00:21.351 DEBUG:teuthology.orchestra.run.smithi031:> sudo systemctl stop ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.1 2023-10-01T15:00:21.644 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 15:00:21 smithi031 systemd[1]: Stopping Ceph osd.1 for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T15:00:21.994 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 15:00:21 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-1[122845]: 2023-10-01T15:00:21.641+0000 7f589105d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T15:00:21.994 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 15:00:21 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-1[122845]: 2023-10-01T15:00:21.641+0000 7f589105d700 -1 osd.1 55 *** Got signal Terminated *** 2023-10-01T15:00:21.994 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 15:00:21 smithi031 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-1[122845]: 2023-10-01T15:00:21.641+0000 7f589105d700 -1 osd.1 55 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T15:00:27.495 INFO:journalctl@ceph.osd.1.smithi031.stdout:Oct 01 15:00:27 smithi031 bash[135258]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-1 2023-10-01T15:00:28.497 DEBUG:teuthology.orchestra.run.smithi031:> sudo pkill -f 'journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-01T15:00:28.541 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T15:00:28.541 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-01T15:00:28.541 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-01T15:00:28.542 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.2 2023-10-01T15:00:28.839 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 15:00:28 smithi099 systemd[1]: Stopping Ceph osd.2 for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T15:00:28.839 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 15:00:28 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-2[113053]: 2023-10-01T15:00:28.787+0000 7ff5e4a14700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T15:00:28.839 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 15:00:28 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-2[113053]: 2023-10-01T15:00:28.787+0000 7ff5e4a14700 -1 osd.2 55 *** Got signal Terminated *** 2023-10-01T15:00:28.839 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 15:00:28 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-2[113053]: 2023-10-01T15:00:28.787+0000 7ff5e4a14700 -1 osd.2 55 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T15:00:34.589 INFO:journalctl@ceph.osd.2.smithi099.stdout:Oct 01 15:00:34 smithi099 bash[124410]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-2 2023-10-01T15:00:35.595 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-01T15:00:35.635 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T15:00:35.635 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-01T15:00:35.635 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-01T15:00:35.635 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.3 2023-10-01T15:00:35.872 INFO:journalctl@ceph.osd.3.smithi099.stdout:Oct 01 15:00:35 smithi099 systemd[1]: Stopping Ceph osd.3 for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T15:00:36.339 INFO:journalctl@ceph.osd.3.smithi099.stdout:Oct 01 15:00:35 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-3[117111]: 2023-10-01T15:00:35.871+0000 7fcf09ae0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T15:00:36.339 INFO:journalctl@ceph.osd.3.smithi099.stdout:Oct 01 15:00:35 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-3[117111]: 2023-10-01T15:00:35.871+0000 7fcf09ae0700 -1 osd.3 55 *** Got signal Terminated *** 2023-10-01T15:00:36.339 INFO:journalctl@ceph.osd.3.smithi099.stdout:Oct 01 15:00:35 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-3[117111]: 2023-10-01T15:00:35.871+0000 7fcf09ae0700 -1 osd.3 55 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T15:00:41.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:40 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:40.728+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:41.839 INFO:journalctl@ceph.osd.3.smithi099.stdout:Oct 01 15:00:41 smithi099 bash[124638]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-3 2023-10-01T15:00:42.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:41 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:41.751+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:42.227 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 15:00:41 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-4[121304]: 2023-10-01T15:00:41.897+0000 7f99f216c700 -1 osd.4 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.242145+0000 front 2023-10-01T15:00:17.242069+0000 (oldest deadline 2023-10-01T15:00:41.342182+0000) 2023-10-01T15:00:42.498 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-01T15:00:42.536 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T15:00:42.536 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-01T15:00:42.537 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-01T15:00:42.537 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.4 2023-10-01T15:00:42.779 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 15:00:42 smithi099 systemd[1]: Stopping Ceph osd.4 for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T15:00:43.088 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 15:00:42 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-4[121304]: 2023-10-01T15:00:42.778+0000 7f99f837b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T15:00:43.089 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 15:00:42 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-4[121304]: 2023-10-01T15:00:42.778+0000 7f99f837b700 -1 osd.4 55 *** Got signal Terminated *** 2023-10-01T15:00:43.089 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 15:00:42 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-4[121304]: 2023-10-01T15:00:42.778+0000 7f99f837b700 -1 osd.4 55 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T15:00:43.089 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 15:00:42 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-4[121304]: 2023-10-01T15:00:42.885+0000 7f99f216c700 -1 osd.4 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.242145+0000 front 2023-10-01T15:00:17.242069+0000 (oldest deadline 2023-10-01T15:00:41.342182+0000) 2023-10-01T15:00:43.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:42 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:42.813+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:43.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:42 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:42.723+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:43.669 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:43 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:43.329+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.319906+0000 front 2023-10-01T15:00:18.319976+0000 (oldest deadline 2023-10-01T15:00:43.019930+0000) 2023-10-01T15:00:44.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:43 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:43.844+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:44.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:43 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:43.720+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:44.338 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 15:00:43 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-4[121304]: 2023-10-01T15:00:43.839+0000 7f99f216c700 -1 osd.4 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.242145+0000 front 2023-10-01T15:00:17.242069+0000 (oldest deadline 2023-10-01T15:00:41.342182+0000) 2023-10-01T15:00:44.669 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:44 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:44.326+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.319906+0000 front 2023-10-01T15:00:18.319976+0000 (oldest deadline 2023-10-01T15:00:43.019930+0000) 2023-10-01T15:00:45.089 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 15:00:44 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-4[121304]: 2023-10-01T15:00:44.796+0000 7f99f216c700 -1 osd.4 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.242145+0000 front 2023-10-01T15:00:17.242069+0000 (oldest deadline 2023-10-01T15:00:41.342182+0000) 2023-10-01T15:00:45.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:44 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:44.867+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:45.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:44 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:44.698+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:45.669 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:45 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:45.293+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.319906+0000 front 2023-10-01T15:00:18.319976+0000 (oldest deadline 2023-10-01T15:00:43.019930+0000) 2023-10-01T15:00:46.089 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 15:00:45 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-4[121304]: 2023-10-01T15:00:45.834+0000 7f99f216c700 -1 osd.4 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.242145+0000 front 2023-10-01T15:00:17.242069+0000 (oldest deadline 2023-10-01T15:00:41.342182+0000) 2023-10-01T15:00:46.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:45 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:45.667+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:46.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:45 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:45.869+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:46.626 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:46 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:46.337+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.319906+0000 front 2023-10-01T15:00:18.319976+0000 (oldest deadline 2023-10-01T15:00:43.019930+0000) 2023-10-01T15:00:46.919 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:46 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:46.824+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:46.919 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:46 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:46.625+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:47.088 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 15:00:46 smithi099 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-4[121304]: 2023-10-01T15:00:46.801+0000 7f99f216c700 -1 osd.4 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.242145+0000 front 2023-10-01T15:00:17.242069+0000 (oldest deadline 2023-10-01T15:00:41.342182+0000) 2023-10-01T15:00:47.669 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:47 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:47.383+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.319906+0000 front 2023-10-01T15:00:18.319976+0000 (oldest deadline 2023-10-01T15:00:43.019930+0000) 2023-10-01T15:00:47.669 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:47 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:47.631+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:47.669 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:47 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:47.631+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:00:48.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:47 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:47.826+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:48.669 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:48 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:48.417+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.319906+0000 front 2023-10-01T15:00:18.319976+0000 (oldest deadline 2023-10-01T15:00:43.019930+0000) 2023-10-01T15:00:48.838 INFO:journalctl@ceph.osd.4.smithi099.stdout:Oct 01 15:00:48 smithi099 bash[124864]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-4 2023-10-01T15:00:49.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:48 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:48.804+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:49.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:48 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:48.804+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:00:49.170 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:48 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:48.678+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:49.170 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:48 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:48.678+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:00:49.646 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-01T15:00:49.685 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T15:00:49.685 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-01T15:00:49.686 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-01T15:00:49.686 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.5 2023-10-01T15:00:49.718 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:49 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:49.460+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.319906+0000 front 2023-10-01T15:00:18.319976+0000 (oldest deadline 2023-10-01T15:00:43.019930+0000) 2023-10-01T15:00:49.718 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:49 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:49.460+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:26.421145+0000 front 2023-10-01T15:00:26.421168+0000 (oldest deadline 2023-10-01T15:00:49.321315+0000) 2023-10-01T15:00:49.719 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:49 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:49.693+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:49.719 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:49 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:49.693+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:00:50.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:49 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:49.804+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:50.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:49 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:49.804+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:00:50.170 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:49 smithi135 systemd[1]: Stopping Ceph osd.5 for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T15:00:50.170 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:49 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:49.911+0000 7fc19e5c8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T15:00:50.170 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:49 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:49.911+0000 7fc19e5c8700 -1 osd.5 55 *** Got signal Terminated *** 2023-10-01T15:00:50.170 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:49 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:49.911+0000 7fc19e5c8700 -1 osd.5 55 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T15:00:50.793 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:50 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:50.462+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.319906+0000 front 2023-10-01T15:00:18.319976+0000 (oldest deadline 2023-10-01T15:00:43.019930+0000) 2023-10-01T15:00:50.793 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:50 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:50.462+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:26.421145+0000 front 2023-10-01T15:00:26.421168+0000 (oldest deadline 2023-10-01T15:00:49.321315+0000) 2023-10-01T15:00:50.794 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:50 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:50.692+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:50.794 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:50 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:50.692+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:00:51.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:50 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:50.791+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:51.170 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:50 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:50.791+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:00:51.815 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:51 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:51.497+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.319906+0000 front 2023-10-01T15:00:18.319976+0000 (oldest deadline 2023-10-01T15:00:43.019930+0000) 2023-10-01T15:00:51.815 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:51 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:51.497+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:26.421145+0000 front 2023-10-01T15:00:26.421168+0000 (oldest deadline 2023-10-01T15:00:49.321315+0000) 2023-10-01T15:00:51.815 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:51 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:51.725+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:51.816 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:51 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:51.725+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:00:52.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:51 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:51.814+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:52.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:51 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:51.814+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:00:52.804 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:52 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:52.527+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.319906+0000 front 2023-10-01T15:00:18.319976+0000 (oldest deadline 2023-10-01T15:00:43.019930+0000) 2023-10-01T15:00:52.805 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:52 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:52.527+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:26.421145+0000 front 2023-10-01T15:00:26.421168+0000 (oldest deadline 2023-10-01T15:00:49.321315+0000) 2023-10-01T15:00:52.805 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:52 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:52.690+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:52.805 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:52 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:52.690+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:00:53.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:52 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:52.802+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:53.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:52 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:52.802+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:00:53.843 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:53 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:53.514+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.319906+0000 front 2023-10-01T15:00:18.319976+0000 (oldest deadline 2023-10-01T15:00:43.019930+0000) 2023-10-01T15:00:53.844 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:53 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:53.514+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:26.421145+0000 front 2023-10-01T15:00:26.421168+0000 (oldest deadline 2023-10-01T15:00:49.321315+0000) 2023-10-01T15:00:53.844 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:53 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:53.726+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:53.844 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:53 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:53.726+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:00:54.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:53 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:53.841+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:54.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:53 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:53.841+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:00:54.872 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:54 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:54.509+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.319906+0000 front 2023-10-01T15:00:18.319976+0000 (oldest deadline 2023-10-01T15:00:43.019930+0000) 2023-10-01T15:00:54.872 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:54 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5[114065]: 2023-10-01T15:00:54.509+0000 7fc198bba700 -1 osd.5 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:26.421145+0000 front 2023-10-01T15:00:26.421168+0000 (oldest deadline 2023-10-01T15:00:49.321315+0000) 2023-10-01T15:00:54.872 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:54 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:54.704+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:54.873 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:54 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:54.704+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:00:55.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:54 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:54.870+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:55.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:54 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:54.870+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:00:55.859 INFO:journalctl@ceph.osd.5.smithi135.stdout:Oct 01 15:00:55 smithi135 bash[125340]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-5 2023-10-01T15:00:55.859 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:55 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:55.694+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:55.859 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:55 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:55.694+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:00:56.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:55 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:55.857+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:56.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:55 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:55.857+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:00:56.771 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:56 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:56.656+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:56.771 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:56 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:56.656+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:00:56.778 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-01T15:00:56.858 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T15:00:56.858 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-01T15:00:56.858 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-01T15:00:56.858 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.6 2023-10-01T15:00:57.097 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:56 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:56.828+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:57.097 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:56 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:56.828+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:00:57.097 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:56 smithi135 systemd[1]: Stopping Ceph osd.6 for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T15:00:57.097 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:57 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:57.095+0000 7fc01811f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T15:00:57.419 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:57 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:57.095+0000 7fc01811f700 -1 osd.6 55 *** Got signal Terminated *** 2023-10-01T15:00:57.419 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:57 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:57.095+0000 7fc01811f700 -1 osd.6 55 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T15:00:58.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:57 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:57.859+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:58.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:57 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:57.859+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:00:58.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:57 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:57.859+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.776847+0000 front 2023-10-01T15:00:33.776836+0000 (oldest deadline 2023-10-01T15:00:57.276748+0000) 2023-10-01T15:00:58.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:57 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:57.672+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:58.170 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:57 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:57.672+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:00:58.919 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:58 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:58.891+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:00:58.919 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:58 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:58.891+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:00:58.919 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:58 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:58.891+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.776847+0000 front 2023-10-01T15:00:33.776836+0000 (oldest deadline 2023-10-01T15:00:57.276748+0000) 2023-10-01T15:00:58.920 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:58 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:58.653+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:58.920 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:58 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:58.653+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:00:58.920 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:58 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:58.653+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.092696+0000 front 2023-10-01T15:00:33.092682+0000 (oldest deadline 2023-10-01T15:00:58.392629+0000) 2023-10-01T15:00:59.918 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:59 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:59.636+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:00:59.918 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:59 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:59.636+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:00:59.919 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:00:59 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:00:59.636+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.092696+0000 front 2023-10-01T15:00:33.092682+0000 (oldest deadline 2023-10-01T15:00:58.392629+0000) 2023-10-01T15:01:00.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:59 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:59.917+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:01:00.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:59 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:59.917+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:01:00.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:00:59 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:00:59.917+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.776847+0000 front 2023-10-01T15:00:33.776836+0000 (oldest deadline 2023-10-01T15:00:57.276748+0000) 2023-10-01T15:01:00.889 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:00 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:00.598+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:01:00.889 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:00 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:00.598+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:01:00.890 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:00 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:00.598+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.092696+0000 front 2023-10-01T15:00:33.092682+0000 (oldest deadline 2023-10-01T15:00:58.392629+0000) 2023-10-01T15:01:01.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:01:00 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:01:00.887+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:01:01.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:01:00 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:01:00.887+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:01:01.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:01:00 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:01:00.887+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.776847+0000 front 2023-10-01T15:00:33.776836+0000 (oldest deadline 2023-10-01T15:00:57.276748+0000) 2023-10-01T15:01:01.850 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:01 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:01.587+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:01:01.850 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:01 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:01.587+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:01:01.850 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:01 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:01.587+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.092696+0000 front 2023-10-01T15:00:33.092682+0000 (oldest deadline 2023-10-01T15:00:58.392629+0000) 2023-10-01T15:01:02.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:01:01 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:01:01.849+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:17.974947+0000 front 2023-10-01T15:00:17.974968+0000 (oldest deadline 2023-10-01T15:00:42.675064+0000) 2023-10-01T15:01:02.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:01:01 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:01:01.849+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.375969+0000 front 2023-10-01T15:00:24.375994+0000 (oldest deadline 2023-10-01T15:00:48.475977+0000) 2023-10-01T15:01:02.169 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:01:01 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6[118304]: 2023-10-01T15:01:01.849+0000 7fc014715700 -1 osd.6 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.776847+0000 front 2023-10-01T15:00:33.776836+0000 (oldest deadline 2023-10-01T15:00:57.276748+0000) 2023-10-01T15:01:02.919 INFO:journalctl@ceph.osd.6.smithi135.stdout:Oct 01 15:01:02 smithi135 bash[125569]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-6 2023-10-01T15:01:02.919 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:02 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:02.619+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:01:02.920 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:02 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:02.619+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:01:02.920 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:02 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:02.619+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.092696+0000 front 2023-10-01T15:00:33.092682+0000 (oldest deadline 2023-10-01T15:00:58.392629+0000) 2023-10-01T15:01:03.919 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:03 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:03.658+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:01:03.919 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:03 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:03.658+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:01:03.919 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:03 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:03.658+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.092696+0000 front 2023-10-01T15:00:33.092682+0000 (oldest deadline 2023-10-01T15:00:58.392629+0000) 2023-10-01T15:01:03.920 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:03 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:03.658+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6812 osd.3 since back 2023-10-01T15:00:39.493823+0000 front 2023-10-01T15:00:39.493775+0000 (oldest deadline 2023-10-01T15:01:03.593540+0000) 2023-10-01T15:01:03.927 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-01T15:01:03.965 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T15:01:03.966 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-01T15:01:03.966 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-01T15:01:03.966 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.7 2023-10-01T15:01:04.205 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:04 smithi135 systemd[1]: Stopping Ceph osd.7 for 466018a4-606a-11ee-8db5-212e2dc638e7... 2023-10-01T15:01:04.649 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:04 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:04.203+0000 7f6e658b6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T15:01:04.649 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:04 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:04.203+0000 7f6e658b6700 -1 osd.7 55 *** Got signal Terminated *** 2023-10-01T15:01:04.649 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:04 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:04.203+0000 7f6e658b6700 -1 osd.7 55 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T15:01:04.919 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:04 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:04.648+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:01:04.919 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:04 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:04.648+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:01:04.919 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:04 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:04.648+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.092696+0000 front 2023-10-01T15:00:33.092682+0000 (oldest deadline 2023-10-01T15:00:58.392629+0000) 2023-10-01T15:01:04.919 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:04 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:04.648+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6812 osd.3 since back 2023-10-01T15:00:39.493823+0000 front 2023-10-01T15:00:39.493775+0000 (oldest deadline 2023-10-01T15:01:03.593540+0000) 2023-10-01T15:01:06.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:05 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:05.688+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:01:06.170 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:05 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:05.688+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:01:06.170 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:05 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:05.688+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.092696+0000 front 2023-10-01T15:00:33.092682+0000 (oldest deadline 2023-10-01T15:00:58.392629+0000) 2023-10-01T15:01:06.170 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:05 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:05.688+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6812 osd.3 since back 2023-10-01T15:00:39.493823+0000 front 2023-10-01T15:00:39.493775+0000 (oldest deadline 2023-10-01T15:01:03.593540+0000) 2023-10-01T15:01:07.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:06 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:06.730+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:01:07.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:06 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:06.730+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:01:07.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:06 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:06.730+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.092696+0000 front 2023-10-01T15:00:33.092682+0000 (oldest deadline 2023-10-01T15:00:58.392629+0000) 2023-10-01T15:01:07.170 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:06 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:06.730+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6812 osd.3 since back 2023-10-01T15:00:39.493823+0000 front 2023-10-01T15:00:39.493775+0000 (oldest deadline 2023-10-01T15:01:03.593540+0000) 2023-10-01T15:01:08.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:07 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:07.696+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:01:08.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:07 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:07.696+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:01:08.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:07 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:07.696+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.092696+0000 front 2023-10-01T15:00:33.092682+0000 (oldest deadline 2023-10-01T15:00:58.392629+0000) 2023-10-01T15:01:08.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:07 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:07.696+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6812 osd.3 since back 2023-10-01T15:00:39.493823+0000 front 2023-10-01T15:00:39.493775+0000 (oldest deadline 2023-10-01T15:01:03.593540+0000) 2023-10-01T15:01:09.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:08 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:08.706+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6806 osd.0 since back 2023-10-01T15:00:18.689518+0000 front 2023-10-01T15:00:18.689602+0000 (oldest deadline 2023-10-01T15:00:40.389538+0000) 2023-10-01T15:01:09.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:08 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:08.706+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.31:6814 osd.1 since back 2023-10-01T15:00:24.490510+0000 front 2023-10-01T15:00:24.490572+0000 (oldest deadline 2023-10-01T15:00:47.390706+0000) 2023-10-01T15:01:09.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:08 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:08.706+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6804 osd.2 since back 2023-10-01T15:00:33.092696+0000 front 2023-10-01T15:00:33.092682+0000 (oldest deadline 2023-10-01T15:00:58.392629+0000) 2023-10-01T15:01:09.169 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:08 smithi135 ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7[122443]: 2023-10-01T15:01:08.706+0000 7f6e61eac700 -1 osd.7 55 heartbeat_check: no reply from 172.21.15.99:6812 osd.3 since back 2023-10-01T15:00:39.493823+0000 front 2023-10-01T15:00:39.493775+0000 (oldest deadline 2023-10-01T15:01:03.593540+0000) 2023-10-01T15:01:09.919 INFO:journalctl@ceph.osd.7.smithi135.stdout:Oct 01 15:01:09 smithi135 bash[125809]: ceph-466018a4-606a-11ee-8db5-212e2dc638e7-osd-7 2023-10-01T15:01:10.923 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-466018a4-606a-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-01T15:01:10.962 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T15:01:10.963 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-01T15:01:10.963 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-01T15:01:11.148 INFO:teuthology.orchestra.run.smithi031.stdout:Deleting cluster with fsid: 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T15:01:12.875 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-01T15:01:13.079 INFO:teuthology.orchestra.run.smithi099.stdout:Deleting cluster with fsid: 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T15:01:13.809 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-01T15:01:13.990 INFO:teuthology.orchestra.run.smithi135.stdout:Deleting cluster with fsid: 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T15:01:14.728 DEBUG:teuthology.orchestra.run.smithi031:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T15:01:14.768 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T15:01:14.804 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T15:01:14.834 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-01T15:01:14.836 DEBUG:teuthology.misc:Transferring archived files from smithi031:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615/remote/smithi031/crash 2023-10-01T15:01:14.837 DEBUG:teuthology.orchestra.run.smithi031:> sudo tar c -f - -C /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/crash -- . 2023-10-01T15:01:14.891 INFO:teuthology.orchestra.run.smithi031.stderr:tar: /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-01T15:01:14.891 INFO:teuthology.orchestra.run.smithi031.stderr:tar: Error is not recoverable: exiting now 2023-10-01T15:01:14.893 DEBUG:teuthology.misc:Transferring archived files from smithi099:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615/remote/smithi099/crash 2023-10-01T15:01:14.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/crash -- . 2023-10-01T15:01:14.946 INFO:teuthology.orchestra.run.smithi099.stderr:tar: /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-01T15:01:14.946 INFO:teuthology.orchestra.run.smithi099.stderr:tar: Error is not recoverable: exiting now 2023-10-01T15:01:14.948 DEBUG:teuthology.misc:Transferring archived files from smithi135:/var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615/remote/smithi135/crash 2023-10-01T15:01:14.949 DEBUG:teuthology.orchestra.run.smithi135:> sudo tar c -f - -C /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/crash -- . 2023-10-01T15:01:14.987 INFO:teuthology.orchestra.run.smithi135.stderr:tar: /var/lib/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-01T15:01:14.988 INFO:teuthology.orchestra.run.smithi135.stderr:tar: Error is not recoverable: exiting now 2023-10-01T15:01:14.988 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-01T15:01:14.989 DEBUG:teuthology.orchestra.run.smithi031:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-01T15:01:15.024 INFO:teuthology.orchestra.run.smithi031.stderr:grep: /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-01T15:01:15.026 INFO:tasks.cephadm:Compressing logs... 2023-10-01T15:01:15.026 DEBUG:teuthology.orchestra.run.smithi031:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2023-10-01T15:01:15.069 DEBUG:teuthology.orchestra.run.smithi099:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2023-10-01T15:01:15.072 DEBUG:teuthology.orchestra.run.smithi135:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2023-10-01T15:01:15.098 INFO:teuthology.orchestra.run.smithi135.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2023-10-01T15:01:15.099 INFO:teuthology.orchestra.run.smithi135.stderr:: No such file or directory 2023-10-01T15:01:15.100 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.6.log 2023-10-01T15:01:15.100 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-volume.log 2023-10-01T15:01:15.100 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.7.log 2023-10-01T15:01:15.100 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.6.log: /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-mon.c.log 2023-10-01T15:01:15.100 INFO:teuthology.orchestra.run.smithi135.stderr: 90.7% -- replaced with /var/log/ceph/cephadm.log.gz 2023-10-01T15:01:15.101 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.5.log 2023-10-01T15:01:15.102 INFO:teuthology.orchestra.run.smithi031.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log : No such file or directory 2023-10-01T15:01:15.102 INFO:teuthology.orchestra.run.smithi031.stderr: 2023-10-01T15:01:15.102 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-mon.c.log: /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.5.log: 91.7% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-volume.log.gz 2023-10-01T15:01:15.105 INFO:teuthology.orchestra.run.smithi031.stderr:gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-volume.log 2023-10-01T15:01:15.105 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.0.log 2023-10-01T15:01:15.105 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-mgr.x.log 2023-10-01T15:01:15.105 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-mon.a.log 2023-10-01T15:01:15.105 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.1.log 2023-10-01T15:01:15.106 INFO:teuthology.orchestra.run.smithi099.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-01T15:01:15.106 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2023-10-01T15:01:15.106 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-volume.log 2023-10-01T15:01:15.106 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.2.log 2023-10-01T15:01:15.106 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.3.log 2023-10-01T15:01:15.107 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-mon.b.log 2023-10-01T15:01:15.107 INFO:teuthology.orchestra.run.smithi099.stderr: 89.8% -- replaced with /var/log/ceph/cephadm.log.gz 2023-10-01T15:01:15.107 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.4.log 2023-10-01T15:01:15.107 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-mon.b.log: /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.4.log: 91.7% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-volume.log.gz 2023-10-01T15:01:15.107 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-mon.a.log: /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.1.log: 91.4% -- replaced with /var/log/ceph/cephadm.log.gz 2023-10-01T15:01:15.108 INFO:teuthology.orchestra.run.smithi031.stderr: 91.7% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-volume.log.gz 2023-10-01T15:01:15.148 INFO:teuthology.orchestra.run.smithi135.stderr: 92.8% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.6.log.gz 2023-10-01T15:01:15.152 INFO:teuthology.orchestra.run.smithi099.stderr: 92.6% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.4.log.gz 2023-10-01T15:01:15.160 INFO:teuthology.orchestra.run.smithi135.stderr: 92.4% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-mon.c.log.gz 2023-10-01T15:01:15.166 INFO:teuthology.orchestra.run.smithi031.stderr: 92.9% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.0.log.gz 2023-10-01T15:01:15.171 INFO:teuthology.orchestra.run.smithi099.stderr: 92.8% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.3.log.gz 2023-10-01T15:01:15.172 INFO:teuthology.orchestra.run.smithi135.stderr: 92.7% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.5.log.gz 2023-10-01T15:01:15.172 INFO:teuthology.orchestra.run.smithi135.stderr: 93.1% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.7.log.gz 2023-10-01T15:01:15.174 INFO:teuthology.orchestra.run.smithi135.stderr: 2023-10-01T15:01:15.174 INFO:teuthology.orchestra.run.smithi135.stderr:real 0m0.085s 2023-10-01T15:01:15.174 INFO:teuthology.orchestra.run.smithi135.stderr:user 0m0.250s 2023-10-01T15:01:15.174 INFO:teuthology.orchestra.run.smithi135.stderr:sys 0m0.028s 2023-10-01T15:01:15.176 INFO:teuthology.orchestra.run.smithi031.stderr: 93.0% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.1.log.gz 2023-10-01T15:01:15.176 INFO:teuthology.orchestra.run.smithi099.stderr: 92.3% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-mon.b.log.gz 2023-10-01T15:01:15.201 INFO:teuthology.orchestra.run.smithi099.stderr: 93.4% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-osd.2.log.gz 2023-10-01T15:01:15.203 INFO:teuthology.orchestra.run.smithi099.stderr: 2023-10-01T15:01:15.203 INFO:teuthology.orchestra.run.smithi099.stderr:real 0m0.112s 2023-10-01T15:01:15.203 INFO:teuthology.orchestra.run.smithi099.stderr:user 0m0.278s 2023-10-01T15:01:15.203 INFO:teuthology.orchestra.run.smithi099.stderr:sys 0m0.036s 2023-10-01T15:01:15.204 INFO:teuthology.orchestra.run.smithi031.stderr: 89.6% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-mgr.x.log.gz 2023-10-01T15:01:15.358 INFO:teuthology.orchestra.run.smithi031.stderr: 91.1% -- replaced with /var/log/ceph/466018a4-606a-11ee-8db5-212e2dc638e7/ceph-mon.a.log.gz 2023-10-01T15:01:15.360 INFO:teuthology.orchestra.run.smithi031.stderr: 2023-10-01T15:01:15.360 INFO:teuthology.orchestra.run.smithi031.stderr:real 0m0.271s 2023-10-01T15:01:15.361 INFO:teuthology.orchestra.run.smithi031.stderr:user 0m0.485s 2023-10-01T15:01:15.361 INFO:teuthology.orchestra.run.smithi031.stderr:sys 0m0.038s 2023-10-01T15:01:15.361 INFO:tasks.cephadm:Archiving logs... 2023-10-01T15:01:15.361 DEBUG:teuthology.misc:Transferring archived files from smithi031:/var/log/ceph to /home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615/remote/smithi031/log 2023-10-01T15:01:15.362 DEBUG:teuthology.orchestra.run.smithi031:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-01T15:01:15.502 DEBUG:teuthology.misc:Transferring archived files from smithi099:/var/log/ceph to /home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615/remote/smithi099/log 2023-10-01T15:01:15.503 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-01T15:01:15.578 DEBUG:teuthology.misc:Transferring archived files from smithi135:/var/log/ceph to /home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615/remote/smithi135/log 2023-10-01T15:01:15.579 DEBUG:teuthology.orchestra.run.smithi135:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-01T15:01:15.643 INFO:tasks.cephadm:Removing cluster... 2023-10-01T15:01:15.644 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 --force 2023-10-01T15:01:15.844 INFO:teuthology.orchestra.run.smithi031.stdout:Deleting cluster with fsid: 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T15:01:16.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 --force 2023-10-01T15:01:16.194 INFO:teuthology.orchestra.run.smithi099.stdout:Deleting cluster with fsid: 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T15:01:16.350 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 466018a4-606a-11ee-8db5-212e2dc638e7 --force 2023-10-01T15:01:16.532 INFO:teuthology.orchestra.run.smithi135.stdout:Deleting cluster with fsid: 466018a4-606a-11ee-8db5-212e2dc638e7 2023-10-01T15:01:16.687 INFO:tasks.cephadm:Removing cephadm ... 2023-10-01T15:01:16.687 DEBUG:teuthology.orchestra.run.smithi031:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-01T15:01:16.711 DEBUG:teuthology.orchestra.run.smithi099:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-01T15:01:16.729 DEBUG:teuthology.orchestra.run.smithi135:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-01T15:01:16.747 INFO:tasks.cephadm:Teardown complete 2023-10-01T15:01:16.747 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-01T15:01:16.759 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-01T15:01:16.759 DEBUG:teuthology.orchestra.run.smithi031:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T15:01:16.762 DEBUG:teuthology.orchestra.run.smithi099:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T15:01:16.771 DEBUG:teuthology.orchestra.run.smithi135:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T15:01:16.780 INFO:teuthology.orchestra.run.smithi031.stderr:bash: ntpq: command not found 2023-10-01T15:01:16.786 INFO:teuthology.orchestra.run.smithi031.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-01T15:01:16.786 INFO:teuthology.orchestra.run.smithi031.stdout:=============================================================================== 2023-10-01T15:01:16.786 INFO:teuthology.orchestra.run.smithi031.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 28 +2207ns[+2207ns] +/- 83ms 2023-10-01T15:01:16.786 INFO:teuthology.orchestra.run.smithi031.stdout:^* hv02.front.sepia.ceph.com 3 6 377 29 -1366us[-1373us] +/- 58ms 2023-10-01T15:01:16.786 INFO:teuthology.orchestra.run.smithi031.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 29 +1911us[+1911us] +/- 78ms 2023-10-01T15:01:16.787 INFO:teuthology.orchestra.run.smithi031.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T15:01:16.787 INFO:teuthology.orchestra.run.smithi099.stderr:bash: ntpq: command not found 2023-10-01T15:01:16.790 INFO:teuthology.orchestra.run.smithi099.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-01T15:01:16.790 INFO:teuthology.orchestra.run.smithi099.stdout:=============================================================================== 2023-10-01T15:01:16.790 INFO:teuthology.orchestra.run.smithi099.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 30 -17us[ +21us] +/- 83ms 2023-10-01T15:01:16.790 INFO:teuthology.orchestra.run.smithi099.stdout:^* hv02.front.sepia.ceph.com 3 6 377 29 -1364us[-1325us] +/- 58ms 2023-10-01T15:01:16.790 INFO:teuthology.orchestra.run.smithi099.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 29 +1911us[+1911us] +/- 78ms 2023-10-01T15:01:16.791 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T15:01:16.803 INFO:teuthology.orchestra.run.smithi135.stderr:bash: ntpq: command not found 2023-10-01T15:01:16.808 INFO:teuthology.orchestra.run.smithi135.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-01T15:01:16.808 INFO:teuthology.orchestra.run.smithi135.stdout:=============================================================================== 2023-10-01T15:01:16.808 INFO:teuthology.orchestra.run.smithi135.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 31 -9914ns[ +15us] +/- 83ms 2023-10-01T15:01:16.808 INFO:teuthology.orchestra.run.smithi135.stdout:^* hv02.front.sepia.ceph.com 3 6 377 29 -1363us[-1339us] +/- 58ms 2023-10-01T15:01:16.808 INFO:teuthology.orchestra.run.smithi135.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 29 +1891us[+1891us] +/- 78ms 2023-10-01T15:01:16.808 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T15:01:16.809 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-01T15:01:16.818 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-01T15:01:16.819 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-01T15:01:16.827 DEBUG:teuthology.orchestra.run.smithi031:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-01T15:01:16.935 DEBUG:teuthology.orchestra.run.smithi099:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-01T15:01:17.025 DEBUG:teuthology.orchestra.run.smithi135:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-01T15:01:17.114 DEBUG:teuthology.orchestra.run.smithi031:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-01T15:01:17.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T15:01:17.153 DEBUG:teuthology.orchestra.run.smithi099:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-01T15:01:17.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T15:01:17.185 DEBUG:teuthology.orchestra.run.smithi135:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-01T15:01:17.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T15:01:17.217 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-01T15:01:17.227 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-01T15:01:17.259 INFO:teuthology.task.internal:Duration was 968.630867 seconds 2023-10-01T15:01:17.259 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-01T15:01:17.268 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-01T15:01:17.268 DEBUG:teuthology.orchestra.run.smithi031:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-01T15:01:17.270 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-01T15:01:17.272 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-01T15:01:17.317 INFO:teuthology.orchestra.run.smithi099.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-01T15:01:17.318 INFO:teuthology.orchestra.run.smithi135.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-01T15:01:17.328 INFO:teuthology.orchestra.run.smithi031.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-01T15:01:17.682 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-01T15:01:17.682 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi031.front.sepia.ceph.com 2023-10-01T15:01:17.682 DEBUG:teuthology.orchestra.run.smithi031:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-01T15:01:17.713 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi099.front.sepia.ceph.com 2023-10-01T15:01:17.713 DEBUG:teuthology.orchestra.run.smithi099:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-01T15:01:17.736 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi135.front.sepia.ceph.com 2023-10-01T15:01:17.736 DEBUG:teuthology.orchestra.run.smithi135:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-01T15:01:17.760 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-01T15:01:17.760 DEBUG:teuthology.orchestra.run.smithi031:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-01T15:01:17.763 DEBUG:teuthology.orchestra.run.smithi099:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-01T15:01:17.779 DEBUG:teuthology.orchestra.run.smithi135:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-01T15:01:17.842 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-01T15:01:17.842 DEBUG:teuthology.orchestra.run.smithi031:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-01T15:01:17.845 DEBUG:teuthology.orchestra.run.smithi099:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-01T15:01:17.859 DEBUG:teuthology.orchestra.run.smithi135:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-01T15:01:17.973 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-01T15:01:17.982 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-01T15:01:17.983 DEBUG:teuthology.orchestra.run.smithi031:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-01T15:01:18.015 DEBUG:teuthology.orchestra.run.smithi099:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-01T15:01:18.043 DEBUG:teuthology.orchestra.run.smithi135:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-01T15:01:18.069 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-01T15:01:18.079 DEBUG:teuthology.orchestra.run.smithi031:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-01T15:01:18.081 DEBUG:teuthology.orchestra.run.smithi099:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-01T15:01:18.085 DEBUG:teuthology.orchestra.run.smithi135:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-01T15:01:18.109 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern = core 2023-10-01T15:01:18.114 INFO:teuthology.orchestra.run.smithi031.stdout:kernel.core_pattern = core 2023-10-01T15:01:18.136 INFO:teuthology.orchestra.run.smithi135.stdout:kernel.core_pattern = core 2023-10-01T15:01:18.181 DEBUG:teuthology.orchestra.run.smithi031:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-01T15:01:18.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T15:01:18.207 DEBUG:teuthology.orchestra.run.smithi099:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-01T15:01:18.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T15:01:18.224 DEBUG:teuthology.orchestra.run.smithi135:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-01T15:01:18.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T15:01:18.242 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-01T15:01:18.251 INFO:teuthology.task.internal:Transferring archived files... 2023-10-01T15:01:18.252 DEBUG:teuthology.misc:Transferring archived files from smithi031:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615/remote/smithi031 2023-10-01T15:01:18.252 DEBUG:teuthology.orchestra.run.smithi031:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-01T15:01:18.300 DEBUG:teuthology.misc:Transferring archived files from smithi099:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615/remote/smithi099 2023-10-01T15:01:18.300 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-01T15:01:18.338 DEBUG:teuthology.misc:Transferring archived files from smithi135:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615/remote/smithi135 2023-10-01T15:01:18.338 DEBUG:teuthology.orchestra.run.smithi135:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-01T15:01:18.375 INFO:teuthology.task.internal:Removing archive directory... 2023-10-01T15:01:18.375 DEBUG:teuthology.orchestra.run.smithi031:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-01T15:01:18.378 DEBUG:teuthology.orchestra.run.smithi099:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-01T15:01:18.379 DEBUG:teuthology.orchestra.run.smithi135:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-01T15:01:18.427 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-01T15:01:18.439 INFO:teuthology.task.internal:Not uploading archives. 2023-10-01T15:01:18.439 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-01T15:01:18.448 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-01T15:01:18.448 DEBUG:teuthology.orchestra.run.smithi031:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-01T15:01:18.451 DEBUG:teuthology.orchestra.run.smithi099:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-01T15:01:18.453 DEBUG:teuthology.orchestra.run.smithi135:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-01T15:01:18.468 INFO:teuthology.orchestra.run.smithi099.stdout: 526080 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 1 15:01 /home/ubuntu/cephtest 2023-10-01T15:01:18.471 INFO:teuthology.orchestra.run.smithi031.stdout: 525357 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 1 15:01 /home/ubuntu/cephtest 2023-10-01T15:01:18.471 INFO:teuthology.orchestra.run.smithi031.stdout: 527037 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 1 14:59 /home/ubuntu/cephtest/mnt.0 2023-10-01T15:01:18.471 INFO:teuthology.orchestra.run.smithi031.stdout: 527038 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 1 15:00 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T15:01:18.472 INFO:teuthology.orchestra.run.smithi031.stdout: 527113 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 1 15:00 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-01T15:01:18.472 INFO:teuthology.orchestra.run.smithi031.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-01T15:01:18.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T15:01:18.473 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi031 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-01T15:01:18.479 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-01T15:01:18.490 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-01T15:01:18.500 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi031 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-01T15:01:18.501 INFO:teuthology.nuke:Checking targets against current locks 2023-10-01T15:01:18.508 INFO:teuthology.orchestra.run.smithi135.stdout: 526080 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 1 15:01 /home/ubuntu/cephtest 2023-10-01T15:01:18.539 DEBUG:teuthology.nuke:shortname: smithi031 2023-10-01T15:01:18.539 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-01T15:01:18.562 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi031.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 14:36:44.222950', 'locked_by': 'scheduled_bdavidov@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBzFxjmohYMjBlhZFHaSqP2EGsaSqCtkeQuECrh7GWlv1Hazs3K0S4MFPSfARFzwjQEVUAI9C+RJBva3kdN9j/k='} 2023-10-01T15:01:18.566 DEBUG:teuthology.nuke:shortname: smithi099 2023-10-01T15:01:18.566 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-01T15:01:18.592 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi099.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 14:36:44.226013', 'locked_by': 'scheduled_bdavidov@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEir+edhJYBvVeyTWMD79z8V5/Di8zfrBuQ/MOReH20F9B4nAfw2XqkY5elDs7LwzCWe4rmEVwLnKH0mppYWTDU='} 2023-10-01T15:01:18.593 DEBUG:teuthology.nuke:shortname: smithi135 2023-10-01T15:01:18.593 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-01T15:01:18.617 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi135.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/bdavidov-2023-09-26_21:57:54-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7402615', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 14:36:44.224882', 'locked_by': 'scheduled_bdavidov@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEhRzE5lRVOrZR+Y6zSbCrvxEQA40nzLw6gCAf4qkfz9R6H2+pkIEjLbJZ4gG+atSdadb2qcDmnnuKaIRokoVdI='} 2023-10-01T15:01:18.626 INFO:teuthology.orchestra.console.smithi031:Power off 2023-10-01T15:01:18.626 DEBUG:teuthology.orchestra.console.smithi031:pexpect command: ipmitool -H smithi031.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-01T15:01:18.653 DEBUG:teuthology.orchestra.console.smithi031:power off output: Chassis Power Control: Down/Off 2023-10-01T15:01:18.653 DEBUG:teuthology.orchestra.console.smithi031:pexpect command: ipmitool -H smithi031.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T15:01:18.666 INFO:teuthology.orchestra.console.smithi099:Power off 2023-10-01T15:01:18.667 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-01T15:01:18.677 DEBUG:teuthology.orchestra.console.smithi031:check power output: Chassis Power is on 2023-10-01T15:01:18.686 DEBUG:teuthology.orchestra.console.smithi099:power off output: Chassis Power Control: Down/Off 2023-10-01T15:01:18.687 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T15:01:18.703 INFO:teuthology.orchestra.console.smithi135:Power off 2023-10-01T15:01:18.703 DEBUG:teuthology.orchestra.console.smithi135:pexpect command: ipmitool -H smithi135.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-01T15:01:18.710 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is on 2023-10-01T15:01:18.723 DEBUG:teuthology.orchestra.console.smithi135:power off output: Chassis Power Control: Down/Off 2023-10-01T15:01:18.723 DEBUG:teuthology.orchestra.console.smithi135:pexpect command: ipmitool -H smithi135.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T15:01:18.746 DEBUG:teuthology.orchestra.console.smithi135:check power output: Chassis Power is on 2023-10-01T15:01:22.678 DEBUG:teuthology.orchestra.console.smithi031:pexpect command: ipmitool -H smithi031.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T15:01:22.713 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T15:01:22.748 DEBUG:teuthology.orchestra.console.smithi135:pexpect command: ipmitool -H smithi135.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T15:01:22.788 DEBUG:teuthology.orchestra.console.smithi031:check power output: Chassis Power is on 2023-10-01T15:01:22.822 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is on 2023-10-01T15:01:22.857 DEBUG:teuthology.orchestra.console.smithi135:check power output: Chassis Power is on 2023-10-01T15:01:26.793 DEBUG:teuthology.orchestra.console.smithi031:pexpect command: ipmitool -H smithi031.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T15:01:26.823 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T15:01:26.858 DEBUG:teuthology.orchestra.console.smithi135:pexpect command: ipmitool -H smithi135.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T15:01:26.904 DEBUG:teuthology.orchestra.console.smithi031:check power output: Chassis Power is on 2023-10-01T15:01:26.932 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is on 2023-10-01T15:01:26.967 DEBUG:teuthology.orchestra.console.smithi135:check power output: Chassis Power is on 2023-10-01T15:01:30.905 DEBUG:teuthology.orchestra.console.smithi031:pexpect command: ipmitool -H smithi031.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T15:01:30.933 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T15:01:30.967 DEBUG:teuthology.orchestra.console.smithi135:pexpect command: ipmitool -H smithi135.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T15:01:31.015 DEBUG:teuthology.orchestra.console.smithi031:check power output: Chassis Power is off 2023-10-01T15:01:31.042 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is off 2023-10-01T15:01:31.075 DEBUG:teuthology.orchestra.console.smithi135:check power output: Chassis Power is off 2023-10-01T15:01:31.116 INFO:teuthology.orchestra.console.smithi031:Power off completed 2023-10-01T15:01:31.143 INFO:teuthology.orchestra.console.smithi099:Power off completed 2023-10-01T15:01:31.175 INFO:teuthology.orchestra.console.smithi135:Power off completed 2023-10-01T15:01:31.280 INFO:teuthology.run:Summary data: description: rbd:nvmeof/{0-single-container-host base/install cluster/{fixed-3 openstack} workloads/cephadm_nvmeof} duration: 968.630866765976 failure_reason: 'Command failed (workunit test cephadm/test_nvme_etc_hosts.sh) on smithi031 with status 127: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=71faca2bb8dc1aef7c81acc0e385f3ca26f84fe5 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_nvme_etc_hosts.sh''' owner: scheduled_bdavidov@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=d852654015ed48f8a464f112d6212444 status: fail success: false 2023-10-01T15:01:31.280 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-01T15:01:31.354 INFO:teuthology.run:FAIL