2023-10-09T07:00:28.851 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-09T07:00:28.852 DEBUG:teuthology.run:Teuthology command: teuthology --name bdavidov-2023-10-09_06:52:16-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi --verbose --owner scheduled_bdavidov@teuthology --description rbd:nvmeof/{0-single-container-host base/install cluster/{fixed-3 openstack} workloads/cephadm_nvmeof} --archive /home/teuthworker/archive/bdavidov-2023-10-09_06:52:16-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7418388 -- /home/teuthworker/archive/bdavidov-2023-10-09_06:52:16-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7418388/orig.config.yaml 2023-10-09T07:00:28.885 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T07:00:28.969 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/bdavidov-2023-10-09_06:52:16-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7418388 branch: wip-barakda-cephadm-nvmeof description: rbd:nvmeof/{0-single-container-host base/install cluster/{fixed-3 openstack} workloads/cephadm_nvmeof} email: null first_in_suite: false job_id: '7418388' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: bdavidov-2023-10-09_06:52:16-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - machine: cpus: 1 disk: 40 ram: 8000 volumes: count: 4 size: 30 os_type: centos os_version: 8.stream overrides: admin_socket: branch: wip-barakda-cephadm-nvmeof ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: bcea8c48517ff154f392584022efa0718f51c9f2 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: bcea8c48517ff154f392584022efa0718f51c9f2 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-nvmetest-teuthology sha1: 1cec30186a27ce6ee457e59e165e4c9eb45c6a8d owner: scheduled_bdavidov@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - mon.a - mgr.x - osd.0 - osd.1 - client.0 seed: 2842 sha1: bcea8c48517ff154f392584022efa0718f51c9f2 sleep_before_teardown: 0 suite: rbd:nvmeof suite_branch: wip-nvmetest-teuthology suite_path: /home/teuthworker/src/github.com_barakda_ceph_1cec30186a27ce6ee457e59e165e4c9eb45c6a8d/qa suite_relpath: qa suite_repo: https://github.com/barakda/ceph suite_sha1: 1cec30186a27ce6ee457e59e165e4c9eb45c6a8d targets: smithi115.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDiUqHywS0xD9nJC4DSbT0+NQ4dCY3SgCS6l903K1h/Xrm6/ERMoYUGRtzWqZyFl9yPJcaHdn6isdZRlEnHlOLw= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - sudo sed -i 's/runtime = "runc"/#runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/#runtime = "crun"/runtime = "crun"/g' /usr/share/containers/containers.conf - cephadm: null - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - ceph osd lspools - ceph osd pool create mypool - ceph osd pool application enable mypool rbd - ceph osd lspools - ceph osd pool ls detail - sleep 60 - ceph config set mgr mgr/cephadm/container_image_nvmeof quay.io/ceph/nvmeof:0.0.4 - sleep 60 - ceph config get mgr mgr/cephadm/container_image_nvmeof - ceph orch apply nvmeof mypool --placement="1 $(hostname)" - ceph orch ps --refresh - ceph versions - ceph -s - ceph orch ls - rbd create mypool/myimage --size 8Gi - rbd ls mypool - 'set -ex # cat /proc/cpuinfo # print nvmeof start up logs s=$(systemctl list-units | grep nvmeof.mypool | awk ''{print $1}'') echo "<---- s variable is [ $s ]---->" PAGER=cat journalctl -u $s ' - cephadm.wait_for_service: service: nvmeof.mypool - exec: host.a: - modprobe nvme-fabrics - 'set -ex # # print nvmeof start up logs s=$(systemctl list-units | grep nvmeof.mypool | awk ''{print $1}'') echo "<---- s variable is [ $s ]---->" PAGER=cat journalctl -u $s GATEWAY=$(echo $s | cut -d "@" -f2- | sed ''s/.\{8\}$//'') echo "<---- GATEWAY is [ $GATEWAY ]---->" IP=$(echo $(hostname -I) | cut -d '' '' -f1) HOSTNAME=$(hostname) IMAGE="quay.io/ceph/nvmeof-cli:0.0.4" POOL="mypool" MIMAGE="myimage" BDEV="mybdev" SERIAL="SPDK00000000000001" NQN="nqn.2016-06.io.spdk:cnode1" PORT="4420" SRPORT="5500" podman run -it $IMAGE --server-address $IP --server-port $SRPORT create_bdev --pool $POOL --image $MIMAGE --bdev $BDEV podman images podman ps podman run -it $IMAGE --server-address $IP --server-port $SRPORT create_subsystem --subnqn $NQN --serial $SERIAL podman run -it $IMAGE --server-address $IP --server-port $SRPORT add_namespace --subnqn $NQN --bdev $BDEV podman run -it $IMAGE --server-address $IP --server-port $SRPORT create_listener -n $NQN -g $GATEWAY -a $IP -s $PORT podman run -it $IMAGE --server-address $IP --server-port $SRPORT add_host --subnqn $NQN --host "*" podman run -it $IMAGE --server-address $IP --server-port $SRPORT get_subsystems nvme discover -t tcp -a $IP -s $PORT nvme connect -t tcp --traddr $IP -s $PORT -n $NQN nvme list ' - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - ceph osd lspools - 'set -ex s=$(ceph orch ps | grep nvmeof.mypool | awk ''{print $4}'') echo $s [ $s = running ] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-10-09_06:52:16 tube: smithi use_shaman: true user: bdavidov verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-09T07:00:28.970 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_barakda_ceph_1cec30186a27ce6ee457e59e165e4c9eb45c6a8d/qa; will attempt to use it 2023-10-09T07:00:28.970 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_barakda_ceph_1cec30186a27ce6ee457e59e165e4c9eb45c6a8d/qa/tasks 2023-10-09T07:00:28.971 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-09T07:00:28.972 INFO:teuthology.task.internal:Checking packages... 2023-10-09T07:00:28.994 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'bcea8c48517ff154f392584022efa0718f51c9f2' 2023-10-09T07:00:28.994 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T07:00:28.995 INFO:teuthology.packaging:ref: None 2023-10-09T07:00:28.995 INFO:teuthology.packaging:tag: None 2023-10-09T07:00:28.995 INFO:teuthology.packaging:branch: wip-barakda-cephadm-nvmeof 2023-10-09T07:00:28.995 INFO:teuthology.packaging:sha1: bcea8c48517ff154f392584022efa0718f51c9f2 2023-10-09T07:00:28.995 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=wip-barakda-cephadm-nvmeof 2023-10-09T07:00:29.142 INFO:teuthology.task.internal:Found packages for ceph version 18.0.0-5817.gbcea8c48 2023-10-09T07:00:29.143 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-09T07:00:29.151 INFO:teuthology.task.internal:no buildpackages task found 2023-10-09T07:00:29.151 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-09T07:00:29.195 INFO:teuthology.task.internal:Saving configuration 2023-10-09T07:00:29.208 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-09T07:00:29.213 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T07:00:29.238 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi115.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/bdavidov-2023-10-09_06:52:16-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7418388', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 06:55:27.570305', 'locked_by': 'scheduled_bdavidov@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDiUqHywS0xD9nJC4DSbT0+NQ4dCY3SgCS6l903K1h/Xrm6/ERMoYUGRtzWqZyFl9yPJcaHdn6isdZRlEnHlOLw='} 2023-10-09T07:00:29.238 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-09T07:00:29.244 INFO:teuthology.task.internal:roles: ubuntu@smithi115.front.sepia.ceph.com - ['host.a', 'mon.a', 'mgr.x', 'osd.0', 'osd.1', 'client.0'] 2023-10-09T07:00:29.244 INFO:teuthology.run_tasks:Running task console_log... 2023-10-09T07:00:29.280 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f40b2db7b80>, signals=[15]) 2023-10-09T07:00:29.280 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-09T07:00:29.286 INFO:teuthology.task.internal:Opening connections... 2023-10-09T07:00:29.286 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi115.front.sepia.ceph.com 2023-10-09T07:00:29.288 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi115.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T07:00:29.357 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-09T07:00:29.364 DEBUG:teuthology.orchestra.run.smithi115:> uname -m 2023-10-09T07:00:29.427 INFO:teuthology.orchestra.run.smithi115.stdout:x86_64 2023-10-09T07:00:29.427 DEBUG:teuthology.orchestra.run.smithi115:> cat /etc/os-release 2023-10-09T07:00:29.488 INFO:teuthology.orchestra.run.smithi115.stdout:NAME="CentOS Stream" 2023-10-09T07:00:29.488 INFO:teuthology.orchestra.run.smithi115.stdout:VERSION="8" 2023-10-09T07:00:29.489 INFO:teuthology.orchestra.run.smithi115.stdout:ID="centos" 2023-10-09T07:00:29.489 INFO:teuthology.orchestra.run.smithi115.stdout:ID_LIKE="rhel fedora" 2023-10-09T07:00:29.489 INFO:teuthology.orchestra.run.smithi115.stdout:VERSION_ID="8" 2023-10-09T07:00:29.489 INFO:teuthology.orchestra.run.smithi115.stdout:PLATFORM_ID="platform:el8" 2023-10-09T07:00:29.489 INFO:teuthology.orchestra.run.smithi115.stdout:PRETTY_NAME="CentOS Stream 8" 2023-10-09T07:00:29.489 INFO:teuthology.orchestra.run.smithi115.stdout:ANSI_COLOR="0;31" 2023-10-09T07:00:29.489 INFO:teuthology.orchestra.run.smithi115.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-10-09T07:00:29.489 INFO:teuthology.orchestra.run.smithi115.stdout:HOME_URL="https://centos.org/" 2023-10-09T07:00:29.489 INFO:teuthology.orchestra.run.smithi115.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T07:00:29.489 INFO:teuthology.orchestra.run.smithi115.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T07:00:29.489 INFO:teuthology.orchestra.run.smithi115.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-10-09T07:00:29.490 INFO:teuthology.lock.ops:Updating smithi115.front.sepia.ceph.com on lock server 2023-10-09T07:00:29.513 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-09T07:00:29.522 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-09T07:00:29.528 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-09T07:00:29.528 DEBUG:teuthology.orchestra.run.smithi115:> test '!' -e /home/ubuntu/cephtest 2023-10-09T07:00:29.547 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-09T07:00:29.554 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-09T07:00:29.554 DEBUG:teuthology.orchestra.run.smithi115:> test -z $(ls -A /var/lib/ceph) 2023-10-09T07:00:29.610 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-09T07:00:29.643 INFO:teuthology.run_tasks:Running task kernel... 2023-10-09T07:00:29.657 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-09T07:00:29.657 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-10-09T07:00:29.658 DEBUG:teuthology.orchestra.run.smithi115:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T07:00:29.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T07:00:29.676 DEBUG:teuthology.orchestra.run.smithi115:> uname -r 2023-10-09T07:00:29.736 INFO:teuthology.orchestra.run.smithi115.stdout:4.18.0-489.el8.x86_64 2023-10-09T07:00:29.736 INFO:teuthology.task.kernel:Running kernel on smithi115: 4.18.0-489.el8.x86_64 2023-10-09T07:00:29.737 DEBUG:teuthology.orchestra.run.smithi115:> sudo yum install -y kernel 2023-10-09T07:00:40.191 INFO:teuthology.orchestra.run.smithi115.stdout:CentOS Stream 8 - AppStream 4.2 MB/s | 33 MB 00:07 2023-10-09T07:01:27.983 INFO:teuthology.orchestra.run.smithi115.stdout:CentOS Stream 8 - BaseOS 1.1 MB/s | 48 MB 00:41 2023-10-09T07:01:35.398 INFO:teuthology.orchestra.run.smithi115.stdout:CentOS Stream 8 - Extras 31 kB/s | 18 kB 00:00 2023-10-09T07:01:35.890 INFO:teuthology.orchestra.run.smithi115.stdout:CentOS Stream 8 - Extras common packages 22 kB/s | 6.9 kB 00:00 2023-10-09T07:01:41.485 INFO:teuthology.orchestra.run.smithi115.stdout:Copr repo for python3-asyncssh owned by ceph 652 B/s | 3.5 kB 00:05 2023-10-09T07:01:43.052 INFO:teuthology.orchestra.run.smithi115.stdout:Extra Packages for Enterprise Linux 11 MB/s | 16 MB 00:01 2023-10-09T07:01:46.827 INFO:teuthology.orchestra.run.smithi115.stdout:lab-extras 407 kB/s | 24 kB 00:00 2023-10-09T07:01:50.292 INFO:teuthology.orchestra.run.smithi115.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-10-09T07:01:50.292 INFO:teuthology.orchestra.run.smithi115.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-10-09T07:01:50.533 INFO:teuthology.orchestra.run.smithi115.stdout:Dependencies resolved. 2023-10-09T07:01:50.534 INFO:teuthology.orchestra.run.smithi115.stdout:================================================================================ 2023-10-09T07:01:50.534 INFO:teuthology.orchestra.run.smithi115.stdout: Package Architecture Version Repository Size 2023-10-09T07:01:50.534 INFO:teuthology.orchestra.run.smithi115.stdout:================================================================================ 2023-10-09T07:01:50.535 INFO:teuthology.orchestra.run.smithi115.stdout:Installing: 2023-10-09T07:01:50.535 INFO:teuthology.orchestra.run.smithi115.stdout: kernel x86_64 4.18.0-514.el8 baseos 10 M 2023-10-09T07:01:50.535 INFO:teuthology.orchestra.run.smithi115.stdout:Installing dependencies: 2023-10-09T07:01:50.535 INFO:teuthology.orchestra.run.smithi115.stdout: kernel-core x86_64 4.18.0-514.el8 baseos 43 M 2023-10-09T07:01:50.535 INFO:teuthology.orchestra.run.smithi115.stdout: kernel-modules x86_64 4.18.0-514.el8 baseos 35 M 2023-10-09T07:01:50.535 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:01:50.535 INFO:teuthology.orchestra.run.smithi115.stdout:Transaction Summary 2023-10-09T07:01:50.535 INFO:teuthology.orchestra.run.smithi115.stdout:================================================================================ 2023-10-09T07:01:50.536 INFO:teuthology.orchestra.run.smithi115.stdout:Install 3 Packages 2023-10-09T07:01:50.536 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:01:50.536 INFO:teuthology.orchestra.run.smithi115.stdout:Total download size: 88 M 2023-10-09T07:01:50.537 INFO:teuthology.orchestra.run.smithi115.stdout:Installed size: 96 M 2023-10-09T07:01:50.537 INFO:teuthology.orchestra.run.smithi115.stdout:Downloading Packages: 2023-10-09T07:01:54.460 INFO:teuthology.orchestra.run.smithi115.stdout:(1/3): kernel-4.18.0-514.el8.x86_64.rpm 2.7 MB/s | 10 MB 00:03 2023-10-09T07:01:56.877 INFO:teuthology.orchestra.run.smithi115.stdout:(2/3): kernel-modules-4.18.0-514.el8.x86_64.rpm 5.8 MB/s | 35 MB 00:06 2023-10-09T07:01:57.585 INFO:teuthology.orchestra.run.smithi115.stdout:(3/3): kernel-core-4.18.0-514.el8.x86_64.rpm 6.2 MB/s | 43 MB 00:06 2023-10-09T07:01:57.585 INFO:teuthology.orchestra.run.smithi115.stdout:-------------------------------------------------------------------------------- 2023-10-09T07:01:57.585 INFO:teuthology.orchestra.run.smithi115.stdout:Total 12 MB/s | 88 MB 00:07 2023-10-09T07:01:58.255 INFO:teuthology.orchestra.run.smithi115.stdout:Running transaction check 2023-10-09T07:01:58.657 INFO:teuthology.orchestra.run.smithi115.stdout:Transaction check succeeded. 2023-10-09T07:01:58.658 INFO:teuthology.orchestra.run.smithi115.stdout:Running transaction test 2023-10-09T07:01:59.874 INFO:teuthology.orchestra.run.smithi115.stdout:Transaction test succeeded. 2023-10-09T07:01:59.896 INFO:teuthology.orchestra.run.smithi115.stdout:Running transaction 2023-10-09T07:02:02.645 INFO:teuthology.orchestra.run.smithi115.stdout: Preparing : 1/1 2023-10-09T07:02:03.204 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : kernel-core-4.18.0-514.el8.x86_64 1/3 2023-10-09T07:02:06.761 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: kernel-core-4.18.0-514.el8.x86_64 1/3 2023-10-09T07:02:07.170 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : kernel-modules-4.18.0-514.el8.x86_64 2/3 2023-10-09T07:02:10.551 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: kernel-modules-4.18.0-514.el8.x86_64 2/3 2023-10-09T07:02:10.850 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : kernel-4.18.0-514.el8.x86_64 3/3 2023-10-09T07:02:32.526 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: kernel-core-4.18.0-514.el8.x86_64 3/3 2023-10-09T07:02:32.547 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: kernel-modules-4.18.0-514.el8.x86_64 3/3 2023-10-09T07:02:32.886 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: kernel-4.18.0-514.el8.x86_64 3/3 2023-10-09T07:02:32.886 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : kernel-4.18.0-514.el8.x86_64 1/3 2023-10-09T07:02:32.886 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : kernel-core-4.18.0-514.el8.x86_64 2/3 2023-10-09T07:02:33.592 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : kernel-modules-4.18.0-514.el8.x86_64 3/3 2023-10-09T07:02:33.592 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:02:33.593 INFO:teuthology.orchestra.run.smithi115.stdout:Installed: 2023-10-09T07:02:33.593 INFO:teuthology.orchestra.run.smithi115.stdout: kernel-4.18.0-514.el8.x86_64 kernel-core-4.18.0-514.el8.x86_64 2023-10-09T07:02:33.593 INFO:teuthology.orchestra.run.smithi115.stdout: kernel-modules-4.18.0-514.el8.x86_64 2023-10-09T07:02:33.593 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:02:33.593 INFO:teuthology.orchestra.run.smithi115.stdout:Complete! 2023-10-09T07:02:33.721 DEBUG:teuthology.orchestra.run.smithi115:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T07:02:33.935 INFO:teuthology.orchestra.run.smithi115.stdout:kernel-4.18.0-514.el8.x86_64 2023-10-09T07:02:33.936 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-514.el8.x86_64 2023-10-09T07:02:33.936 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-514.el8.x86_64 2023-10-09T07:02:33.936 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-09T07:02:33.936 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-09T07:02:33.936 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2023-10-09T07:02:33.936 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi115.front.sepia.ceph.com, path=None, version=distro) 2023-10-09T07:02:33.937 DEBUG:teuthology.orchestra.run.smithi115:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T07:02:34.145 INFO:teuthology.orchestra.run.smithi115.stdout:kernel-4.18.0-514.el8.x86_64 2023-10-09T07:02:34.146 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-514.el8.x86_64 2023-10-09T07:02:34.146 DEBUG:teuthology.orchestra.run.smithi115:> sudo rpm -qi grub2-tools 2023-10-09T07:02:34.179 INFO:teuthology.orchestra.run.smithi115.stdout:Name : grub2-tools 2023-10-09T07:02:34.179 INFO:teuthology.orchestra.run.smithi115.stdout:Epoch : 1 2023-10-09T07:02:34.180 INFO:teuthology.orchestra.run.smithi115.stdout:Version : 2.02 2023-10-09T07:02:34.180 INFO:teuthology.orchestra.run.smithi115.stdout:Release : 129.el8 2023-10-09T07:02:34.180 INFO:teuthology.orchestra.run.smithi115.stdout:Architecture: x86_64 2023-10-09T07:02:34.180 INFO:teuthology.orchestra.run.smithi115.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-10-09T07:02:34.180 INFO:teuthology.orchestra.run.smithi115.stdout:Group : System Environment/Base 2023-10-09T07:02:34.180 INFO:teuthology.orchestra.run.smithi115.stdout:Size : 9265586 2023-10-09T07:02:34.180 INFO:teuthology.orchestra.run.smithi115.stdout:License : GPLv3+ 2023-10-09T07:02:34.180 INFO:teuthology.orchestra.run.smithi115.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-10-09T07:02:34.180 INFO:teuthology.orchestra.run.smithi115.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-10-09T07:02:34.180 INFO:teuthology.orchestra.run.smithi115.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-10-09T07:02:34.181 INFO:teuthology.orchestra.run.smithi115.stdout:Build Host : kbuilder.bsys.centos.org 2023-10-09T07:02:34.181 INFO:teuthology.orchestra.run.smithi115.stdout:Relocations : (not relocatable) 2023-10-09T07:02:34.181 INFO:teuthology.orchestra.run.smithi115.stdout:Packager : CentOS BuildSystem 2023-10-09T07:02:34.181 INFO:teuthology.orchestra.run.smithi115.stdout:Vendor : CentOS 2023-10-09T07:02:34.181 INFO:teuthology.orchestra.run.smithi115.stdout:URL : http://www.gnu.org/software/grub/ 2023-10-09T07:02:34.181 INFO:teuthology.orchestra.run.smithi115.stdout:Summary : Support tools for GRUB. 2023-10-09T07:02:34.181 INFO:teuthology.orchestra.run.smithi115.stdout:Description : 2023-10-09T07:02:34.181 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:02:34.181 INFO:teuthology.orchestra.run.smithi115.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-10-09T07:02:34.182 INFO:teuthology.orchestra.run.smithi115.stdout:customizable bootloader with modular architecture. It supports a rich 2023-10-09T07:02:34.182 INFO:teuthology.orchestra.run.smithi115.stdout:variety of kernel formats, file systems, computer architectures and 2023-10-09T07:02:34.182 INFO:teuthology.orchestra.run.smithi115.stdout:hardware devices. 2023-10-09T07:02:34.182 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:02:34.182 INFO:teuthology.orchestra.run.smithi115.stdout:This subpackage provides tools for support of all platforms. 2023-10-09T07:02:34.183 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-10-09T07:02:34.183 INFO:teuthology.task.kernel:Updating grub on smithi115 to boot 4.18.0-514.el8.x86_64 2023-10-09T07:02:34.183 DEBUG:teuthology.orchestra.run.smithi115:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-10-09T07:02:35.598 INFO:teuthology.orchestra.run.smithi115.stderr:Generating grub configuration file ... 2023-10-09T07:02:38.104 INFO:teuthology.orchestra.run.smithi115.stderr:done 2023-10-09T07:02:38.106 DEBUG:teuthology.orchestra.run.smithi115:> mktemp 2023-10-09T07:02:38.139 INFO:teuthology.orchestra.run.smithi115.stdout:/tmp/tmp.X8cxny0VtC 2023-10-09T07:02:38.139 DEBUG:teuthology.orchestra.run.smithi115:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.X8cxny0VtC 2023-10-09T07:02:38.232 DEBUG:teuthology.orchestra.run.smithi115:> sudo chmod 0666 /tmp/tmp.X8cxny0VtC 2023-10-09T07:02:38.331 DEBUG:teuthology.orchestra.remote:smithi115:/tmp/tmp.X8cxny0VtC is 6KB 2023-10-09T07:02:38.378 DEBUG:teuthology.orchestra.run.smithi115:> rm -fr /tmp/tmp.X8cxny0VtC 2023-10-09T07:02:38.398 DEBUG:teuthology.orchestra.run.smithi115:> sudo /bin/ls /boot/loader/entries || true 2023-10-09T07:02:38.469 INFO:teuthology.orchestra.run.smithi115.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-10-09T07:02:38.469 INFO:teuthology.orchestra.run.smithi115.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-10-09T07:02:38.469 INFO:teuthology.orchestra.run.smithi115.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-10-09T07:02:38.469 INFO:teuthology.orchestra.run.smithi115.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-514.el8.x86_64.conf 2023-10-09T07:02:38.470 DEBUG:teuthology.orchestra.run.smithi115:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-514.el8.x86_64 2023-10-09T07:02:38.619 DEBUG:teuthology.orchestra.run.smithi115:> sudo shutdown -r now 2023-10-09T07:03:08.650 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2023-10-09T07:03:08.651 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi115.front.sepia.ceph.com' 2023-10-09T07:03:08.651 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi115.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T07:03:27.098 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.115 2023-10-09T07:03:36.103 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi115.front.sepia.ceph.com' 2023-10-09T07:03:36.103 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi115.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T07:03:39.162 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.115 2023-10-09T07:03:51.174 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi115.front.sepia.ceph.com' 2023-10-09T07:03:51.175 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi115.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T07:04:09.598 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.115 2023-10-09T07:04:24.610 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi115.front.sepia.ceph.com' 2023-10-09T07:04:24.611 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi115.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T07:04:25.098 DEBUG:teuthology.orchestra.run.smithi115:> true 2023-10-09T07:04:25.467 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi115.front.sepia.ceph.com' 2023-10-09T07:04:25.467 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "4.18.0-514.el8.x86_64"... 2023-10-09T07:04:25.467 DEBUG:teuthology.orchestra.run.smithi115:> uname -r 2023-10-09T07:04:25.485 INFO:teuthology.orchestra.run.smithi115.stdout:4.18.0-514.el8.x86_64 2023-10-09T07:04:25.485 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-514.el8.x86_64 vs 4.18.0-514.el8.x86_64 2023-10-09T07:04:25.485 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-09T07:04:25.485 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-10-09T07:04:26.486 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2023-10-09T07:04:26.487 DEBUG:teuthology.orchestra.run.smithi115:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T07:04:26.557 INFO:teuthology.orchestra.run.smithi115.stdout:ttyS1 2023-10-09T07:04:26.574 DEBUG:teuthology.parallel:result is None 2023-10-09T07:04:26.575 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-09T07:04:26.582 INFO:teuthology.task.internal:Creating test directory... 2023-10-09T07:04:26.582 DEBUG:teuthology.orchestra.run.smithi115:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T07:04:26.599 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-09T07:04:26.606 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-09T07:04:26.614 INFO:teuthology.task.internal:Creating archive directory... 2023-10-09T07:04:26.614 DEBUG:teuthology.orchestra.run.smithi115:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T07:04:26.663 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-09T07:04:26.670 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-09T07:04:26.670 DEBUG:teuthology.orchestra.run.smithi115:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T07:04:26.731 INFO:teuthology.orchestra.run.smithi115.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T07:04:26.741 INFO:teuthology.orchestra.run.smithi115.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T07:04:26.742 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-09T07:04:26.749 INFO:teuthology.task.internal:Configuring sudo... 2023-10-09T07:04:26.750 DEBUG:teuthology.orchestra.run.smithi115:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T07:04:26.811 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-09T07:04:26.821 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-09T07:04:26.821 DEBUG:teuthology.orchestra.run.smithi115:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T07:04:26.869 DEBUG:teuthology.orchestra.run.smithi115:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T07:04:26.955 DEBUG:teuthology.orchestra.run.smithi115:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T07:04:27.006 DEBUG:teuthology.orchestra.run.smithi115:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T07:04:27.089 DEBUG:teuthology.orchestra.run.smithi115:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T07:04:27.132 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2023-10-09T07:04:27.133 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T07:04:27.200 DEBUG:teuthology.orchestra.run.smithi115:> sudo service rsyslog restart 2023-10-09T07:04:27.300 INFO:teuthology.orchestra.run.smithi115.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T07:04:27.501 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-09T07:04:27.510 INFO:teuthology.task.internal:Starting timer... 2023-10-09T07:04:27.510 INFO:teuthology.run_tasks:Running task pcp... 2023-10-09T07:04:27.520 INFO:teuthology.run_tasks:Running task selinux... 2023-10-09T07:04:27.528 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-10-09T07:04:27.529 DEBUG:teuthology.orchestra.run.smithi115:> sudo service auditd rotate 2023-10-09T07:04:27.602 INFO:teuthology.orchestra.run.smithi115.stdout:Rotating logs: 2023-10-09T07:04:27.604 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-09T07:04:27.604 DEBUG:teuthology.orchestra.run.smithi115:> /usr/sbin/getenforce 2023-10-09T07:04:27.656 INFO:teuthology.orchestra.run.smithi115.stdout:Permissive 2023-10-09T07:04:27.656 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi115.front.sepia.ceph.com': 'permissive'} 2023-10-09T07:04:27.656 DEBUG:teuthology.orchestra.run.smithi115:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T07:04:27.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T07:04:27.683 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-09T07:04:27.683 DEBUG:teuthology.orchestra.run.smithi115:> sudo /usr/sbin/setenforce permissive 2023-10-09T07:04:27.750 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-09T07:04:27.760 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-10-09T07:04:27.766 INFO:teuthology.repo_utils:Fetching main from origin 2023-10-09T07:04:27.887 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-09T07:04:27.903 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-09T07:04:27.904 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi115.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-09T07:11:49.491 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi115.front.sepia.ceph.com')] 2023-10-09T07:11:49.492 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi115.front.sepia.ceph.com' 2023-10-09T07:11:49.493 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi115.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T07:11:49.564 DEBUG:teuthology.orchestra.run.smithi115:> true 2023-10-09T07:11:49.633 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi115.front.sepia.ceph.com' 2023-10-09T07:11:49.633 INFO:teuthology.run_tasks:Running task clock... 2023-10-09T07:11:49.643 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-09T07:11:49.644 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T07:11:49.644 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T07:11:49.705 INFO:teuthology.orchestra.run.smithi115.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T07:11:49.719 INFO:teuthology.orchestra.run.smithi115.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T07:11:49.777 INFO:teuthology.orchestra.run.smithi115.stderr:sudo: ntpd: command not found 2023-10-09T07:11:49.788 INFO:teuthology.orchestra.run.smithi115.stdout:506 Cannot talk to daemon 2023-10-09T07:11:49.803 INFO:teuthology.orchestra.run.smithi115.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T07:11:49.816 INFO:teuthology.orchestra.run.smithi115.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T07:11:49.886 INFO:teuthology.orchestra.run.smithi115.stderr:bash: ntpq: command not found 2023-10-09T07:11:49.890 INFO:teuthology.orchestra.run.smithi115.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T07:11:49.890 INFO:teuthology.orchestra.run.smithi115.stdout:=============================================================================== 2023-10-09T07:11:49.890 INFO:teuthology.orchestra.run.smithi115.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T07:11:49.890 INFO:teuthology.orchestra.run.smithi115.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T07:11:49.890 INFO:teuthology.orchestra.run.smithi115.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T07:11:49.891 INFO:teuthology.orchestra.run.smithi115.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T07:11:49.891 INFO:teuthology.run_tasks:Running task pexec... 2023-10-09T07:11:49.900 INFO:teuthology.task.pexec:Executing custom commands... 2023-10-09T07:11:49.900 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi115.front.sepia.ceph.com 2023-10-09T07:11:49.900 DEBUG:teuthology.orchestra.run.smithi115:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T07:11:50.810 INFO:teuthology.orchestra.run.smithi115.stdout:Last metadata expiration check: 0:01:35 ago on Mon 09 Oct 2023 07:10:15 AM UTC. 2023-10-09T07:11:52.858 INFO:teuthology.orchestra.run.smithi115.stdout:Dependencies resolved. 2023-10-09T07:11:52.859 INFO:teuthology.orchestra.run.smithi115.stdout:================================================================================ 2023-10-09T07:11:52.859 INFO:teuthology.orchestra.run.smithi115.stdout: Package Architecture Version Repository Size 2023-10-09T07:11:52.859 INFO:teuthology.orchestra.run.smithi115.stdout:================================================================================ 2023-10-09T07:11:52.859 INFO:teuthology.orchestra.run.smithi115.stdout:Resetting modules: 2023-10-09T07:11:52.859 INFO:teuthology.orchestra.run.smithi115.stdout: container-tools 2023-10-09T07:11:52.859 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:11:52.859 INFO:teuthology.orchestra.run.smithi115.stdout:Transaction Summary 2023-10-09T07:11:52.859 INFO:teuthology.orchestra.run.smithi115.stdout:================================================================================ 2023-10-09T07:11:52.859 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:11:53.113 INFO:teuthology.orchestra.run.smithi115.stdout:Complete! 2023-10-09T07:11:54.036 INFO:teuthology.orchestra.run.smithi115.stdout:Last metadata expiration check: 0:01:38 ago on Mon 09 Oct 2023 07:10:15 AM UTC. 2023-10-09T07:11:56.020 INFO:teuthology.orchestra.run.smithi115.stdout:Dependencies resolved. 2023-10-09T07:11:56.175 INFO:teuthology.orchestra.run.smithi115.stdout:================================================================================ 2023-10-09T07:11:56.175 INFO:teuthology.orchestra.run.smithi115.stdout: Package Arch Version Repository Size 2023-10-09T07:11:56.175 INFO:teuthology.orchestra.run.smithi115.stdout:================================================================================ 2023-10-09T07:11:56.175 INFO:teuthology.orchestra.run.smithi115.stdout:Installing group/module packages: 2023-10-09T07:11:56.175 INFO:teuthology.orchestra.run.smithi115.stdout: aardvark-dns x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 1.0 M 2023-10-09T07:11:56.175 INFO:teuthology.orchestra.run.smithi115.stdout: buildah x86_64 1:1.31.3-1.module_el8+664+4072b3ae appstream 8.8 M 2023-10-09T07:11:56.175 INFO:teuthology.orchestra.run.smithi115.stdout: cockpit-podman noarch 75-1.module_el8+664+4072b3ae appstream 739 k 2023-10-09T07:11:56.176 INFO:teuthology.orchestra.run.smithi115.stdout: crun x86_64 1.8.7-1.module_el8+661+d1afb926 appstream 239 k 2023-10-09T07:11:56.176 INFO:teuthology.orchestra.run.smithi115.stdout: netavark x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 3.7 M 2023-10-09T07:11:56.176 INFO:teuthology.orchestra.run.smithi115.stdout: python3-podman noarch 4.6.0-1.module_el8+582+a2558ae2 appstream 177 k 2023-10-09T07:11:56.176 INFO:teuthology.orchestra.run.smithi115.stdout: skopeo x86_64 2:1.13.3-1.module_el8+664+4072b3ae appstream 8.1 M 2023-10-09T07:11:56.176 INFO:teuthology.orchestra.run.smithi115.stdout: toolbox x86_64 0.0.99.4-5.module_el8+649+e2deac1d appstream 2.5 M 2023-10-09T07:11:56.176 INFO:teuthology.orchestra.run.smithi115.stdout: udica noarch 0.2.6-20.module_el8+487+8e42a277 appstream 53 k 2023-10-09T07:11:56.176 INFO:teuthology.orchestra.run.smithi115.stdout:Installing dependencies: 2023-10-09T07:11:56.176 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2023-10-09T07:11:56.176 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2023-10-09T07:11:56.176 INFO:teuthology.orchestra.run.smithi115.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2023-10-09T07:11:56.176 INFO:teuthology.orchestra.run.smithi115.stdout:Installing weak dependencies: 2023-10-09T07:11:56.176 INFO:teuthology.orchestra.run.smithi115.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2023-10-09T07:11:56.176 INFO:teuthology.orchestra.run.smithi115.stdout:Installing module profiles: 2023-10-09T07:11:56.177 INFO:teuthology.orchestra.run.smithi115.stdout: container-tools/common 2023-10-09T07:11:56.177 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:11:56.177 INFO:teuthology.orchestra.run.smithi115.stdout:Enabling module streams: 2023-10-09T07:11:56.177 INFO:teuthology.orchestra.run.smithi115.stdout: container-tools rhel8 2023-10-09T07:11:56.177 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:11:56.177 INFO:teuthology.orchestra.run.smithi115.stdout:Transaction Summary 2023-10-09T07:11:56.177 INFO:teuthology.orchestra.run.smithi115.stdout:================================================================================ 2023-10-09T07:11:56.177 INFO:teuthology.orchestra.run.smithi115.stdout:Install 13 Packages 2023-10-09T07:11:56.177 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:11:56.178 INFO:teuthology.orchestra.run.smithi115.stdout:Total download size: 26 M 2023-10-09T07:11:56.178 INFO:teuthology.orchestra.run.smithi115.stdout:Installed size: 87 M 2023-10-09T07:11:56.178 INFO:teuthology.orchestra.run.smithi115.stdout:Downloading Packages: 2023-10-09T07:11:56.918 INFO:teuthology.orchestra.run.smithi115.stdout:(1/13): aardvark-dns-1.7.0-1.module_el8+487+8e4 1.8 MB/s | 1.0 MB 00:00 2023-10-09T07:11:56.973 INFO:teuthology.orchestra.run.smithi115.stdout:(2/13): cockpit-podman-75-1.module_el8+664+4072 1.2 MB/s | 739 kB 00:00 2023-10-09T07:11:57.002 INFO:teuthology.orchestra.run.smithi115.stdout:(3/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 453 kB/s | 38 kB 00:00 2023-10-09T07:11:57.277 INFO:teuthology.orchestra.run.smithi115.stdout:(4/13): crun-1.8.7-1.module_el8+661+d1afb926.x8 786 kB/s | 239 kB 00:00 2023-10-09T07:11:57.469 INFO:teuthology.orchestra.run.smithi115.stdout:(5/13): python3-podman-4.6.0-1.module_el8+582+a 925 kB/s | 177 kB 00:00 2023-10-09T07:11:57.541 INFO:teuthology.orchestra.run.smithi115.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 343 kB/s | 25 kB 00:00 2023-10-09T07:11:57.644 INFO:teuthology.orchestra.run.smithi115.stdout:(7/13): python3-pyxdg-0.25-16.el8.noarch.rpm 911 kB/s | 94 kB 00:00 2023-10-09T07:11:59.195 INFO:teuthology.orchestra.run.smithi115.stdout:(8/13): netavark-1.7.0-1.module_el8+487+8e42a27 1.7 MB/s | 3.7 MB 00:02 2023-10-09T07:11:59.762 INFO:teuthology.orchestra.run.smithi115.stdout:(9/13): buildah-1.31.3-1.module_el8+664+4072b3a 2.6 MB/s | 8.8 MB 00:03 2023-10-09T07:11:59.849 INFO:teuthology.orchestra.run.smithi115.stdout:(10/13): udica-0.2.6-20.module_el8+487+8e42a277 603 kB/s | 53 kB 00:00 2023-10-09T07:11:59.904 INFO:teuthology.orchestra.run.smithi115.stdout:(11/13): yajl-2.1.0-12.el8.x86_64.rpm 756 kB/s | 41 kB 00:00 2023-10-09T07:11:59.954 INFO:teuthology.orchestra.run.smithi115.stdout:(12/13): toolbox-0.0.99.4-5.module_el8+649+e2de 3.3 MB/s | 2.5 MB 00:00 2023-10-09T07:12:00.079 INFO:teuthology.orchestra.run.smithi115.stdout:(13/13): skopeo-1.13.3-1.module_el8+664+4072b3a 3.3 MB/s | 8.1 MB 00:02 2023-10-09T07:12:00.080 INFO:teuthology.orchestra.run.smithi115.stdout:-------------------------------------------------------------------------------- 2023-10-09T07:12:00.080 INFO:teuthology.orchestra.run.smithi115.stdout:Total 6.6 MB/s | 26 MB 00:03 2023-10-09T07:12:00.349 INFO:teuthology.orchestra.run.smithi115.stdout:Running transaction check 2023-10-09T07:12:00.369 INFO:teuthology.orchestra.run.smithi115.stdout:Transaction check succeeded. 2023-10-09T07:12:00.370 INFO:teuthology.orchestra.run.smithi115.stdout:Running transaction test 2023-10-09T07:12:00.616 INFO:teuthology.orchestra.run.smithi115.stdout:Transaction test succeeded. 2023-10-09T07:12:00.619 INFO:teuthology.orchestra.run.smithi115.stdout:Running transaction 2023-10-09T07:12:01.306 INFO:teuthology.orchestra.run.smithi115.stdout: Preparing : 1/1 2023-10-09T07:12:01.509 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2023-10-09T07:12:01.680 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/13 2023-10-09T07:12:01.877 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/13 2023-10-09T07:12:02.091 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-10-09T07:12:02.633 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 5/13 2023-10-09T07:12:03.597 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-10-09T07:12:03.854 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 7/13 2023-10-09T07:12:04.085 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : crun-1.8.7-1.module_el8+661+d1afb926.x86_64 8/13 2023-10-09T07:12:04.333 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 9/13 2023-10-09T07:12:04.743 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : udica-0.2.6-20.module_el8+487+8e42a277.noarch 10/13 2023-10-09T07:12:05.609 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-10-09T07:12:05.820 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 12/13 2023-10-09T07:12:05.995 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 13/13 2023-10-09T07:12:06.422 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 13/13 2023-10-09T07:12:06.422 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 1/13 2023-10-09T07:12:06.422 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 2/13 2023-10-09T07:12:06.422 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 3/13 2023-10-09T07:12:06.422 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-10-09T07:12:06.422 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : crun-1.8.7-1.module_el8+661+d1afb926.x86_64 5/13 2023-10-09T07:12:06.423 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-10-09T07:12:06.423 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 7/13 2023-10-09T07:12:06.423 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2023-10-09T07:12:06.423 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2023-10-09T07:12:06.423 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 10/13 2023-10-09T07:12:06.423 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-10-09T07:12:06.423 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : udica-0.2.6-20.module_el8+487+8e42a277.noarch 12/13 2023-10-09T07:12:07.134 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2023-10-09T07:12:07.134 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:12:07.134 INFO:teuthology.orchestra.run.smithi115.stdout:Installed: 2023-10-09T07:12:07.134 INFO:teuthology.orchestra.run.smithi115.stdout: aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-10-09T07:12:07.134 INFO:teuthology.orchestra.run.smithi115.stdout: buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 2023-10-09T07:12:07.134 INFO:teuthology.orchestra.run.smithi115.stdout: cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 2023-10-09T07:12:07.134 INFO:teuthology.orchestra.run.smithi115.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2023-10-09T07:12:07.134 INFO:teuthology.orchestra.run.smithi115.stdout: crun-1.8.7-1.module_el8+661+d1afb926.x86_64 2023-10-09T07:12:07.134 INFO:teuthology.orchestra.run.smithi115.stdout: netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-10-09T07:12:07.134 INFO:teuthology.orchestra.run.smithi115.stdout: python3-podman-4.6.0-1.module_el8+582+a2558ae2.noarch 2023-10-09T07:12:07.135 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-10-09T07:12:07.135 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-10-09T07:12:07.135 INFO:teuthology.orchestra.run.smithi115.stdout: skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 2023-10-09T07:12:07.135 INFO:teuthology.orchestra.run.smithi115.stdout: toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 2023-10-09T07:12:07.135 INFO:teuthology.orchestra.run.smithi115.stdout: udica-0.2.6-20.module_el8+487+8e42a277.noarch 2023-10-09T07:12:07.135 INFO:teuthology.orchestra.run.smithi115.stdout: yajl-2.1.0-12.el8.x86_64 2023-10-09T07:12:07.135 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:12:07.135 INFO:teuthology.orchestra.run.smithi115.stdout:Complete! 2023-10-09T07:12:07.420 DEBUG:teuthology.parallel:result is None 2023-10-09T07:12:07.420 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-09T07:12:07.498 INFO:tasks.cephadm:Config: {'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bcea8c48517ff154f392584022efa0718f51c9f2'} 2023-10-09T07:12:07.499 INFO:tasks.cephadm:Cluster image is quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 2023-10-09T07:12:07.499 INFO:tasks.cephadm:Cluster fsid is 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:12:07.499 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-09T07:12:07.499 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.115'} 2023-10-09T07:12:07.499 INFO:tasks.cephadm:First mon is mon.a on smithi115 2023-10-09T07:12:07.500 INFO:tasks.cephadm:First mgr is x 2023-10-09T07:12:07.500 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-09T07:12:07.500 DEBUG:teuthology.orchestra.run.smithi115:> sudo hostname $(hostname -s) 2023-10-09T07:12:07.528 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra 2023-10-09T07:12:07.528 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=bcea8c48517ff154f392584022efa0718f51c9f2 2023-10-09T07:12:07.672 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': 'bcea8c48517ff154f392584022efa0718f51c9f2', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-new-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/72782/', 'root_build_cause': 'SCMTRIGGER', 'version': '18.0.0-5817-gbcea8c48', 'node_name': '172.21.4.65+confusa03', 'job_name': 'ceph-dev-new-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic', 'package_manager_version': '18.0.0-5817.gbcea8c48'}, 'url': 'https://2.chacra.ceph.com/r/ceph/wip-barakda-cephadm-nvmeof/bcea8c48517ff154f392584022efa0718f51c9f2/centos/8/flavors/default/', 'distro_codename': None, 'modified': '2023-09-25 21:06:38.915379', 'distro_version': '8', 'project': 'ceph', 'flavor': 'default', 'ref': 'wip-barakda-cephadm-nvmeof', 'chacra_url': 'https://2.chacra.ceph.com/repos/ceph/wip-barakda-cephadm-nvmeof/bcea8c48517ff154f392584022efa0718f51c9f2/centos/8/flavors/default/', 'archs': ['x86_64', 'source', 'arm64'], 'distro': 'centos'}] 2023-10-09T07:12:07.886 INFO:tasks.util.chacra:got chacra host 2.chacra.ceph.com, ref wip-barakda-cephadm-nvmeof, sha1 bcea8c48517ff154f392584022efa0718f51c9f2 from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F8%2Fx86_64&flavor=default&sha1=bcea8c48517ff154f392584022efa0718f51c9f2 2023-10-09T07:12:07.887 INFO:tasks.cephadm:Discovered cachra url: https://2.chacra.ceph.com/binaries/ceph/wip-barakda-cephadm-nvmeof/bcea8c48517ff154f392584022efa0718f51c9f2/centos/8/x86_64/flavors/default/cephadm 2023-10-09T07:12:07.887 DEBUG:teuthology.orchestra.run.smithi115:> curl --silent -L https://2.chacra.ceph.com/binaries/ceph/wip-barakda-cephadm-nvmeof/bcea8c48517ff154f392584022efa0718f51c9f2/centos/8/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T07:12:08.413 INFO:teuthology.orchestra.run.smithi115.stdout:-rw-rw-r--. 1 ubuntu ubuntu 705688 Oct 9 07:12 /home/ubuntu/cephtest/cephadm 2023-10-09T07:12:08.414 DEBUG:teuthology.orchestra.run.smithi115:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T07:12:08.441 INFO:tasks.cephadm:Pulling image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 on all hosts... 2023-10-09T07:12:08.441 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 pull 2023-10-09T07:12:10.090 INFO:teuthology.orchestra.run.smithi115.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2... 2023-10-09T07:12:35.406 INFO:teuthology.orchestra.run.smithi115.stdout:{ 2023-10-09T07:12:35.406 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_version": "ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev)", 2023-10-09T07:12:35.406 INFO:teuthology.orchestra.run.smithi115.stdout: "image_id": "fa922c4449069cae14b17e9ffaef6cf64d1d7080cea061a20dc1d8f318e65863", 2023-10-09T07:12:35.406 INFO:teuthology.orchestra.run.smithi115.stdout: "repo_digests": [ 2023-10-09T07:12:35.407 INFO:teuthology.orchestra.run.smithi115.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:34fad0ea24358428207f7cbc4d6027320c5a86fd3de6f51aaec11e44cfc648d7" 2023-10-09T07:12:35.407 INFO:teuthology.orchestra.run.smithi115.stdout: ] 2023-10-09T07:12:35.407 INFO:teuthology.orchestra.run.smithi115.stdout:} 2023-10-09T07:12:35.425 DEBUG:teuthology.orchestra.run.smithi115:> sudo mkdir -p /etc/ceph 2023-10-09T07:12:35.454 DEBUG:teuthology.orchestra.run.smithi115:> sudo chmod 777 /etc/ceph 2023-10-09T07:12:35.522 INFO:tasks.cephadm:Writing seed config... 2023-10-09T07:12:35.523 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-09T07:12:35.523 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-09T07:12:35.523 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-09T07:12:35.523 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-09T07:12:35.523 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-09T07:12:35.523 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-09T07:12:35.524 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-09T07:12:35.524 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2023-10-09T07:12:35.524 DEBUG:teuthology.orchestra.run.smithi115:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-09T07:12:35.580 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 2817f006-6673-11ee-8db5-212e2dc638e7 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-09T07:12:35.580 DEBUG:teuthology.orchestra.run.smithi115:mon.a> sudo journalctl -f -n 0 -u ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-09T07:12:35.622 DEBUG:teuthology.orchestra.run.smithi115:mgr.x> sudo journalctl -f -n 0 -u ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-09T07:12:35.649 INFO:journalctl@ceph.mon.a.smithi115.stdout:-- Logs begin at Mon 2023-10-09 07:04:09 UTC. -- 2023-10-09T07:12:35.650 INFO:tasks.cephadm:Bootstrapping... 2023-10-09T07:12:35.650 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 -v bootstrap --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id x --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.115 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-09T07:12:35.676 INFO:journalctl@ceph.mgr.x.smithi115.stdout:-- Logs begin at Mon 2023-10-09 07:04:09 UTC. -- 2023-10-09T07:12:35.810 INFO:teuthology.orchestra.run.smithi115.stdout:-------------------------------------------------------------------------------- 2023-10-09T07:12:35.810 INFO:teuthology.orchestra.run.smithi115.stdout:cephadm ['--image', 'quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2', '-v', 'bootstrap', '--fsid', '2817f006-6673-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'x', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.115', '--skip-admin-label'] 2023-10-09T07:12:35.841 INFO:teuthology.orchestra.run.smithi115.stdout:/bin/podman: stdout 4.6.1 2023-10-09T07:12:35.841 INFO:teuthology.orchestra.run.smithi115.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-10-09T07:12:35.841 INFO:teuthology.orchestra.run.smithi115.stdout:Verifying podman|docker is present... 2023-10-09T07:12:35.873 INFO:teuthology.orchestra.run.smithi115.stdout:/bin/podman: stdout 4.6.1 2023-10-09T07:12:35.873 INFO:teuthology.orchestra.run.smithi115.stdout:Verifying lvm2 is present... 2023-10-09T07:12:35.873 INFO:teuthology.orchestra.run.smithi115.stdout:Verifying time synchronization is in place... 2023-10-09T07:12:35.880 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T07:12:35.880 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T07:12:35.886 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T07:12:35.886 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout inactive 2023-10-09T07:12:35.891 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout enabled 2023-10-09T07:12:35.897 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout active 2023-10-09T07:12:35.897 INFO:teuthology.orchestra.run.smithi115.stdout:Unit chronyd.service is enabled and running 2023-10-09T07:12:35.898 INFO:teuthology.orchestra.run.smithi115.stdout:Repeating the final host check... 2023-10-09T07:12:35.931 INFO:teuthology.orchestra.run.smithi115.stdout:/bin/podman: stdout 4.6.1 2023-10-09T07:12:35.931 INFO:teuthology.orchestra.run.smithi115.stdout:podman (/bin/podman) version 4.6.1 is present 2023-10-09T07:12:35.931 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl is present 2023-10-09T07:12:35.931 INFO:teuthology.orchestra.run.smithi115.stdout:lvcreate is present 2023-10-09T07:12:35.937 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T07:12:35.937 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T07:12:35.943 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T07:12:35.943 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout inactive 2023-10-09T07:12:35.949 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout enabled 2023-10-09T07:12:35.955 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout active 2023-10-09T07:12:35.955 INFO:teuthology.orchestra.run.smithi115.stdout:Unit chronyd.service is enabled and running 2023-10-09T07:12:35.955 INFO:teuthology.orchestra.run.smithi115.stdout:Host looks OK 2023-10-09T07:12:35.955 INFO:teuthology.orchestra.run.smithi115.stdout:Cluster fsid: 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:12:35.955 INFO:teuthology.orchestra.run.smithi115.stdout:Acquiring lock 140326423805344 on /run/cephadm/2817f006-6673-11ee-8db5-212e2dc638e7.lock 2023-10-09T07:12:35.955 INFO:teuthology.orchestra.run.smithi115.stdout:Lock 140326423805344 acquired on /run/cephadm/2817f006-6673-11ee-8db5-212e2dc638e7.lock 2023-10-09T07:12:35.956 INFO:teuthology.orchestra.run.smithi115.stdout:Verifying IP 172.21.15.115 port 3300 ... 2023-10-09T07:12:35.956 INFO:teuthology.orchestra.run.smithi115.stdout:Verifying IP 172.21.15.115 port 6789 ... 2023-10-09T07:12:35.956 INFO:teuthology.orchestra.run.smithi115.stdout:Base mon IP(s) is [172.21.15.115:3300, 172.21.15.115:6789], mon addrv is [v2:172.21.15.115:3300,v1:172.21.15.115:6789] 2023-10-09T07:12:35.959 INFO:teuthology.orchestra.run.smithi115.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.115 metric 100 2023-10-09T07:12:35.959 INFO:teuthology.orchestra.run.smithi115.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.115 metric 100 2023-10-09T07:12:35.962 INFO:teuthology.orchestra.run.smithi115.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-10-09T07:12:35.962 INFO:teuthology.orchestra.run.smithi115.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2023-10-09T07:12:35.962 INFO:teuthology.orchestra.run.smithi115.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1778sec hoplimit 64 pref medium 2023-10-09T07:12:35.965 INFO:teuthology.orchestra.run.smithi115.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-09T07:12:35.965 INFO:teuthology.orchestra.run.smithi115.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-10-09T07:12:35.965 INFO:teuthology.orchestra.run.smithi115.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T07:12:35.965 INFO:teuthology.orchestra.run.smithi115.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-10-09T07:12:35.965 INFO:teuthology.orchestra.run.smithi115.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:ce8f/64 scope link 2023-10-09T07:12:35.965 INFO:teuthology.orchestra.run.smithi115.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T07:12:35.966 INFO:teuthology.orchestra.run.smithi115.stdout:Mon IP `172.21.15.115` is in CIDR network `172.21.0.0/20` 2023-10-09T07:12:35.966 INFO:teuthology.orchestra.run.smithi115.stdout:Mon IP `172.21.15.115` is in CIDR network `172.21.0.0/20` 2023-10-09T07:12:35.966 INFO:teuthology.orchestra.run.smithi115.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-10-09T07:12:35.966 INFO:teuthology.orchestra.run.smithi115.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-09T07:12:35.967 INFO:teuthology.orchestra.run.smithi115.stdout:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2... 2023-10-09T07:12:36.350 INFO:teuthology.orchestra.run.smithi115.stdout:/bin/podman: stdout fa922c4449069cae14b17e9ffaef6cf64d1d7080cea061a20dc1d8f318e65863 2023-10-09T07:12:36.350 INFO:teuthology.orchestra.run.smithi115.stdout:/bin/podman: stderr Trying to pull quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2... 2023-10-09T07:12:36.350 INFO:teuthology.orchestra.run.smithi115.stdout:/bin/podman: stderr Getting image source signatures 2023-10-09T07:12:36.350 INFO:teuthology.orchestra.run.smithi115.stdout:/bin/podman: stderr Copying blob sha256:556fc22cf1e739705d2013195476c145689a6e17c5a9071e0d49fdc6e036642a 2023-10-09T07:12:36.350 INFO:teuthology.orchestra.run.smithi115.stdout:/bin/podman: stderr Copying blob sha256:3f4fd5c3c800c6da9ef94691df203a05a5bbbf32c483d09d620ec06eef42ea2b 2023-10-09T07:12:36.350 INFO:teuthology.orchestra.run.smithi115.stdout:/bin/podman: stderr Copying config sha256:fa922c4449069cae14b17e9ffaef6cf64d1d7080cea061a20dc1d8f318e65863 2023-10-09T07:12:36.350 INFO:teuthology.orchestra.run.smithi115.stdout:/bin/podman: stderr Writing manifest to image destination 2023-10-09T07:12:36.958 INFO:teuthology.orchestra.run.smithi115.stdout:ceph: stdout ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev) 2023-10-09T07:12:36.958 INFO:teuthology.orchestra.run.smithi115.stdout:Ceph version: ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev) 2023-10-09T07:12:36.958 INFO:teuthology.orchestra.run.smithi115.stdout:Extracting ceph user uid/gid from container image... 2023-10-09T07:12:37.485 INFO:teuthology.orchestra.run.smithi115.stdout:stat: stdout 167 167 2023-10-09T07:12:37.486 INFO:teuthology.orchestra.run.smithi115.stdout:Creating initial keys... 2023-10-09T07:12:37.998 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph-authtool: stdout AQDlpyNlkhEsJxAAgm7F+VeYeI1qUY8Yr0uisQ== 2023-10-09T07:12:38.523 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph-authtool: stdout AQDmpyNlAUF0ChAA6wIWEfiSbHeMehUAZRf0Dg== 2023-10-09T07:12:39.068 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph-authtool: stdout AQDmpyNlZO/QKhAAJItZ3Cc/EHCI2x7A0NWvSA== 2023-10-09T07:12:39.068 INFO:teuthology.orchestra.run.smithi115.stdout:Creating initial monmap... 2023-10-09T07:12:39.624 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T07:12:39.624 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2023-10-09T07:12:39.624 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:12:39.624 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T07:12:39.624 INFO:teuthology.orchestra.run.smithi115.stdout:monmaptool for a [v2:172.21.15.115:3300,v1:172.21.15.115:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T07:12:39.624 INFO:teuthology.orchestra.run.smithi115.stdout:setting min_mon_release = pacific 2023-10-09T07:12:39.624 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/monmaptool: set fsid to 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:12:39.624 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T07:12:39.624 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:12:39.624 INFO:teuthology.orchestra.run.smithi115.stdout:Creating mon... 2023-10-09T07:12:40.189 INFO:teuthology.orchestra.run.smithi115.stdout:create mon.a on 2023-10-09T07:12:40.396 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-10-09T07:12:40.515 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-2817f006-6673-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-2817f006-6673-11ee-8db5-212e2dc638e7.target. 2023-10-09T07:12:40.515 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-2817f006-6673-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-2817f006-6673-11ee-8db5-212e2dc638e7.target. 2023-10-09T07:12:40.692 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T07:12:40.693 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stderr Failed to reset failed state of unit ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-09T07:12:40.783 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-2817f006-6673-11ee-8db5-212e2dc638e7.target.wants/ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-2817f006-6673-11ee-8db5-212e2dc638e7@.service. 2023-10-09T07:12:41.145 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T07:12:41.145 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout disabled 2023-10-09T07:12:41.155 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T07:12:41.155 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout inactive 2023-10-09T07:12:41.155 INFO:teuthology.orchestra.run.smithi115.stdout:firewalld.service is not enabled 2023-10-09T07:12:41.155 INFO:teuthology.orchestra.run.smithi115.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T07:12:41.155 INFO:teuthology.orchestra.run.smithi115.stdout:Waiting for mon to start... 2023-10-09T07:12:41.156 INFO:teuthology.orchestra.run.smithi115.stdout:Waiting for mon... 2023-10-09T07:12:41.466 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:41 smithi115 ceph-mon[107678]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T07:12:41.910 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout cluster: 2023-10-09T07:12:41.910 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout id: 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:12:41.910 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-10-09T07:12:41.910 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 2023-10-09T07:12:41.910 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout services: 2023-10-09T07:12:41.910 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.394654s) 2023-10-09T07:12:41.910 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-10-09T07:12:41.910 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-10-09T07:12:41.911 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 2023-10-09T07:12:41.911 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout data: 2023-10-09T07:12:41.911 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-10-09T07:12:41.911 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-10-09T07:12:41.911 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-10-09T07:12:41.911 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout pgs: 2023-10-09T07:12:41.911 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 2023-10-09T07:12:41.911 INFO:teuthology.orchestra.run.smithi115.stdout:mon is available 2023-10-09T07:12:41.911 INFO:teuthology.orchestra.run.smithi115.stdout:Assimilating anything we can from ceph.conf... 2023-10-09T07:12:42.368 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:42 smithi115 ceph-mon[107678]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T07:12:42.369 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:42 smithi115 ceph-mon[107678]: monmap e1: 1 mons at {a=[v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0]} removed_ranks: {} 2023-10-09T07:12:42.369 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:42 smithi115 ceph-mon[107678]: fsmap 2023-10-09T07:12:42.369 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:42 smithi115 ceph-mon[107678]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T07:12:42.369 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:42 smithi115 ceph-mon[107678]: mgrmap e1: no daemons active 2023-10-09T07:12:42.369 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:42 smithi115 ceph-mon[107678]: from='client.? 172.21.15.115:0/3568226478' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T07:12:42.701 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 2023-10-09T07:12:42.701 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T07:12:42.701 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout fsid = 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:12:42.701 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.115:3300,v1:172.21.15.115:6789] 2023-10-09T07:12:42.702 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T07:12:42.702 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T07:12:42.702 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T07:12:42.702 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T07:12:42.702 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 2023-10-09T07:12:42.702 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T07:12:42.702 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T07:12:42.702 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 2023-10-09T07:12:42.702 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T07:12:42.702 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T07:12:42.702 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T07:12:42.702 INFO:teuthology.orchestra.run.smithi115.stdout:Generating new minimal ceph.conf... 2023-10-09T07:12:43.402 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:43 smithi115 ceph-mon[107678]: from='client.? 172.21.15.115:0/559072234' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T07:12:43.402 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:43 smithi115 ceph-mon[107678]: from='client.? 172.21.15.115:0/559072234' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T07:12:43.402 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:43 smithi115 ceph-mon[107678]: from='client.? 172.21.15.115:0/940487933' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:12:43.467 INFO:teuthology.orchestra.run.smithi115.stdout:Restarting the monitor... 2023-10-09T07:12:43.687 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:43 smithi115 systemd[1]: Stopping Ceph mon.a for 2817f006-6673-11ee-8db5-212e2dc638e7... 2023-10-09T07:12:43.687 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:43 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mon-a[107674]: 2023-10-09T07:12:43.650+0000 7fc13530d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T07:12:43.687 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:43 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mon-a[107674]: 2023-10-09T07:12:43.650+0000 7fc13530d700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T07:12:44.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:43 smithi115 bash[107930]: ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T07:12:44.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 systemd[1]: ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T07:12:44.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 systemd[1]: Stopped Ceph mon.a for 2817f006-6673-11ee-8db5-212e2dc638e7. 2023-10-09T07:12:44.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 systemd[1]: Starting Ceph mon.a for 2817f006-6673-11ee-8db5-212e2dc638e7... 2023-10-09T07:12:44.492 INFO:teuthology.orchestra.run.smithi115.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2023-10-09T07:12:44.887 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 bash[108043]: bb6211c91791aabb639c0c23e905f4e44fadd5f48f5efd78b316848e50b14d45 2023-10-09T07:12:44.887 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 systemd[1]: Started Ceph mon.a for 2817f006-6673-11ee-8db5-212e2dc638e7. 2023-10-09T07:12:44.887 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T07:12:44.887 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev), process ceph-mon, pid 2 2023-10-09T07:12:44.887 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: pidfile_write: ignore empty --pid-file 2023-10-09T07:12:44.888 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: load: jerasure load: lrc 2023-10-09T07:12:44.888 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: RocksDB version: 7.9.2 2023-10-09T07:12:44.888 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Git sha 0 2023-10-09T07:12:44.888 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Compile date 2023-09-25 19:01:54 2023-10-09T07:12:44.888 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: DB SUMMARY 2023-10-09T07:12:44.888 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: DB Session ID: BD119NLW9VHJNIMIPX3K 2023-10-09T07:12:44.888 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: CURRENT file: CURRENT 2023-10-09T07:12:44.888 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T07:12:44.889 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2023-10-09T07:12:44.889 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-10-09T07:12:44.889 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 82838 ; 2023-10-09T07:12:44.889 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.error_if_exists: 0 2023-10-09T07:12:44.889 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.create_if_missing: 0 2023-10-09T07:12:44.889 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.paranoid_checks: 1 2023-10-09T07:12:44.890 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T07:12:44.890 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T07:12:44.890 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T07:12:44.890 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.env: 0x55a8bfa9fda0 2023-10-09T07:12:44.890 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T07:12:44.890 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.info_log: 0x55a8c13b9460 2023-10-09T07:12:44.890 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T07:12:44.890 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.statistics: (nil) 2023-10-09T07:12:44.890 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.use_fsync: 0 2023-10-09T07:12:44.890 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_log_file_size: 0 2023-10-09T07:12:44.890 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T07:12:44.891 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T07:12:44.891 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T07:12:44.891 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T07:12:44.891 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.allow_fallocate: 1 2023-10-09T07:12:44.891 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T07:12:44.891 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T07:12:44.891 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.use_direct_reads: 0 2023-10-09T07:12:44.891 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T07:12:44.891 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T07:12:44.892 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.db_log_dir: 2023-10-09T07:12:44.892 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.wal_dir: 2023-10-09T07:12:44.892 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T07:12:44.892 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T07:12:44.892 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T07:12:44.892 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T07:12:44.892 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T07:12:44.892 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T07:12:44.892 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T07:12:44.892 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T07:12:44.893 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.write_buffer_manager: 0x55a8c13734a0 2023-10-09T07:12:44.893 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T07:12:44.893 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T07:12:44.893 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T07:12:44.893 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T07:12:44.893 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T07:12:44.893 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T07:12:44.893 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T07:12:44.893 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T07:12:44.893 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.unordered_write: 0 2023-10-09T07:12:44.894 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T07:12:44.894 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T07:12:44.894 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T07:12:44.894 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T07:12:44.894 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.row_cache: None 2023-10-09T07:12:44.895 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.wal_filter: None 2023-10-09T07:12:44.895 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T07:12:44.895 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T07:12:44.895 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.two_write_queues: 0 2023-10-09T07:12:44.895 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T07:12:44.895 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.wal_compression: 0 2023-10-09T07:12:44.896 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.atomic_flush: 0 2023-10-09T07:12:44.896 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T07:12:44.896 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T07:12:44.896 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T07:12:44.896 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.log_readahead_size: 0 2023-10-09T07:12:44.896 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T07:12:44.896 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T07:12:44.897 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T07:12:44.897 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T07:12:44.897 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T07:12:44.897 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T07:12:44.897 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T07:12:44.897 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_background_jobs: 2 2023-10-09T07:12:44.897 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_background_compactions: -1 2023-10-09T07:12:44.898 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_subcompactions: 1 2023-10-09T07:12:44.898 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T07:12:44.898 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T07:12:44.898 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T07:12:44.898 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T07:12:44.898 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T07:12:44.899 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T07:12:44.899 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T07:12:44.899 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T07:12:44.899 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_open_files: -1 2023-10-09T07:12:44.899 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T07:12:44.899 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T07:12:44.899 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T07:12:44.899 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T07:12:44.900 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_background_flushes: -1 2023-10-09T07:12:44.900 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Compression algorithms supported: 2023-10-09T07:12:44.900 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T07:12:44.900 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: kZSTD supported: 0 2023-10-09T07:12:44.900 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: kXpressCompression supported: 0 2023-10-09T07:12:44.900 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T07:12:44.901 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: kZlibCompression supported: 1 2023-10-09T07:12:44.901 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: kSnappyCompression supported: 1 2023-10-09T07:12:44.901 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: kLZ4Compression supported: 1 2023-10-09T07:12:44.901 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: kBZip2Compression supported: 0 2023-10-09T07:12:44.901 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T07:12:44.901 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T07:12:44.902 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2023-10-09T07:12:44.902 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T07:12:44.902 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T07:12:44.902 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.merge_operator: 2023-10-09T07:12:44.902 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compaction_filter: None 2023-10-09T07:12:44.902 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T07:12:44.902 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T07:12:44.902 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T07:12:44.903 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T07:12:44.903 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a8c13b9560) 2023-10-09T07:12:44.903 INFO:journalctl@ceph.mon.a.smithi115.stdout: cache_index_and_filter_blocks: 1 2023-10-09T07:12:44.903 INFO:journalctl@ceph.mon.a.smithi115.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T07:12:44.903 INFO:journalctl@ceph.mon.a.smithi115.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T07:12:44.903 INFO:journalctl@ceph.mon.a.smithi115.stdout: pin_top_level_index_and_filter: 1 2023-10-09T07:12:44.903 INFO:journalctl@ceph.mon.a.smithi115.stdout: index_type: 0 2023-10-09T07:12:44.904 INFO:journalctl@ceph.mon.a.smithi115.stdout: data_block_index_type: 0 2023-10-09T07:12:44.904 INFO:journalctl@ceph.mon.a.smithi115.stdout: index_shortening: 1 2023-10-09T07:12:44.904 INFO:journalctl@ceph.mon.a.smithi115.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T07:12:44.904 INFO:journalctl@ceph.mon.a.smithi115.stdout: checksum: 4 2023-10-09T07:12:44.904 INFO:journalctl@ceph.mon.a.smithi115.stdout: no_block_cache: 0 2023-10-09T07:12:44.904 INFO:journalctl@ceph.mon.a.smithi115.stdout: block_cache: 0x55a8c14111f0 2023-10-09T07:12:44.905 INFO:journalctl@ceph.mon.a.smithi115.stdout: block_cache_name: BinnedLRUCache 2023-10-09T07:12:44.905 INFO:journalctl@ceph.mon.a.smithi115.stdout: block_cache_options: 2023-10-09T07:12:44.905 INFO:journalctl@ceph.mon.a.smithi115.stdout: capacity : 536870912 2023-10-09T07:12:44.905 INFO:journalctl@ceph.mon.a.smithi115.stdout: num_shard_bits : 4 2023-10-09T07:12:44.905 INFO:journalctl@ceph.mon.a.smithi115.stdout: strict_capacity_limit : 0 2023-10-09T07:12:44.905 INFO:journalctl@ceph.mon.a.smithi115.stdout: high_pri_pool_ratio: 0.000 2023-10-09T07:12:44.906 INFO:journalctl@ceph.mon.a.smithi115.stdout: block_cache_compressed: (nil) 2023-10-09T07:12:44.906 INFO:journalctl@ceph.mon.a.smithi115.stdout: persistent_cache: (nil) 2023-10-09T07:12:44.906 INFO:journalctl@ceph.mon.a.smithi115.stdout: block_size: 4096 2023-10-09T07:12:44.906 INFO:journalctl@ceph.mon.a.smithi115.stdout: block_size_deviation: 10 2023-10-09T07:12:44.906 INFO:journalctl@ceph.mon.a.smithi115.stdout: block_restart_interval: 16 2023-10-09T07:12:44.906 INFO:journalctl@ceph.mon.a.smithi115.stdout: index_block_restart_interval: 1 2023-10-09T07:12:44.907 INFO:journalctl@ceph.mon.a.smithi115.stdout: metadata_block_size: 4096 2023-10-09T07:12:44.907 INFO:journalctl@ceph.mon.a.smithi115.stdout: partition_filters: 0 2023-10-09T07:12:44.907 INFO:journalctl@ceph.mon.a.smithi115.stdout: use_delta_encoding: 1 2023-10-09T07:12:44.907 INFO:journalctl@ceph.mon.a.smithi115.stdout: filter_policy: bloomfilter 2023-10-09T07:12:44.907 INFO:journalctl@ceph.mon.a.smithi115.stdout: whole_key_filtering: 1 2023-10-09T07:12:44.907 INFO:journalctl@ceph.mon.a.smithi115.stdout: verify_compression: 0 2023-10-09T07:12:44.908 INFO:journalctl@ceph.mon.a.smithi115.stdout: read_amp_bytes_per_bit: 0 2023-10-09T07:12:44.908 INFO:journalctl@ceph.mon.a.smithi115.stdout: format_version: 5 2023-10-09T07:12:44.908 INFO:journalctl@ceph.mon.a.smithi115.stdout: enable_index_compression: 1 2023-10-09T07:12:44.908 INFO:journalctl@ceph.mon.a.smithi115.stdout: block_align: 0 2023-10-09T07:12:44.908 INFO:journalctl@ceph.mon.a.smithi115.stdout: max_auto_readahead_size: 262144 2023-10-09T07:12:44.908 INFO:journalctl@ceph.mon.a.smithi115.stdout: prepopulate_block_cache: 0 2023-10-09T07:12:44.908 INFO:journalctl@ceph.mon.a.smithi115.stdout: initial_auto_readahead_size: 8192 2023-10-09T07:12:44.908 INFO:journalctl@ceph.mon.a.smithi115.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T07:12:44.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T07:12:44.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T07:12:44.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compression: NoCompression 2023-10-09T07:12:44.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T07:12:44.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T07:12:44.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T07:12:44.910 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.num_levels: 7 2023-10-09T07:12:44.910 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T07:12:44.910 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T07:12:44.910 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T07:12:44.910 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T07:12:44.911 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T07:12:44.911 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T07:12:44.911 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T07:12:44.911 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T07:12:44.911 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T07:12:44.912 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T07:12:44.912 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T07:12:44.912 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T07:12:44.912 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T07:12:44.912 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T07:12:44.912 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T07:12:44.912 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T07:12:44.912 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T07:12:44.912 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T07:12:44.912 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T07:12:44.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T07:12:44.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T07:12:44.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T07:12:44.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T07:12:44.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T07:12:44.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T07:12:44.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T07:12:44.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T07:12:44.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T07:12:44.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T07:12:44.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T07:12:44.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T07:12:44.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T07:12:44.914 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T07:12:44.914 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T07:12:44.914 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T07:12:44.914 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T07:12:44.914 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T07:12:44.914 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T07:12:44.914 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T07:12:44.914 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T07:12:44.914 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T07:12:44.914 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T07:12:44.915 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T07:12:44.915 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T07:12:44.915 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T07:12:44.915 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T07:12:44.915 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T07:12:44.915 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T07:12:44.915 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T07:12:44.915 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T07:12:44.915 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T07:12:44.915 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T07:12:44.915 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T07:12:44.916 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.table_properties_collectors: 2023-10-09T07:12:44.916 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.inplace_update_support: 0 2023-10-09T07:12:44.916 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T07:12:44.916 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T07:12:44.916 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T07:12:44.916 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T07:12:44.916 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.bloom_locality: 0 2023-10-09T07:12:44.916 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.max_successive_merges: 0 2023-10-09T07:12:44.916 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T07:12:44.916 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T07:12:44.916 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T07:12:44.916 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T07:12:44.917 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.ttl: 2592000 2023-10-09T07:12:44.917 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T07:12:44.917 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T07:12:44.917 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T07:12:44.917 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.enable_blob_files: false 2023-10-09T07:12:44.917 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.min_blob_size: 0 2023-10-09T07:12:44.917 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T07:12:44.917 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T07:12:44.917 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T07:12:44.917 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T07:12:44.917 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T07:12:44.917 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T07:12:44.918 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T07:12:44.918 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T07:12:44.918 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2023-10-09T07:12:44.918 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2023-10-09T07:12:44.918 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f36fc322-f216-461d-ae35-4b967ae2f679 2023-10-09T07:12:44.918 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696835564498202, "job": 1, "event": "recovery_started", "wal_files": [9]} 2023-10-09T07:12:44.918 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2023-10-09T07:12:44.918 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696835564499281, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 79763, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 242, "table_properties": {"data_size": 77943, "index_size": 230, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9724, "raw_average_key_size": 47, "raw_value_size": 72386, "raw_average_value_size": 353, "num_data_blocks": 10, "num_entries": 205, "num_filter_entries": 205, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696835564, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f36fc322-f216-461d-ae35-4b967ae2f679", "db_session_id": "BD119NLW9VHJNIMIPX3K", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2023-10-09T07:12:44.918 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696835564499360, "job": 1, "event": "recovery_finished"} 2023-10-09T07:12:44.918 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2023-10-09T07:12:44.919 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T07:12:44.919 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a8c14ae000 2023-10-09T07:12:44.919 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: DB pointer 0x55a8c149a000 2023-10-09T07:12:44.919 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T07:12:44.919 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T07:12:44.919 INFO:journalctl@ceph.mon.a.smithi115.stdout: ** DB Stats ** 2023-10-09T07:12:44.919 INFO:journalctl@ceph.mon.a.smithi115.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T07:12:44.919 INFO:journalctl@ceph.mon.a.smithi115.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T07:12:44.919 INFO:journalctl@ceph.mon.a.smithi115.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T07:12:44.919 INFO:journalctl@ceph.mon.a.smithi115.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T07:12:44.919 INFO:journalctl@ceph.mon.a.smithi115.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T07:12:44.919 INFO:journalctl@ceph.mon.a.smithi115.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T07:12:44.920 INFO:journalctl@ceph.mon.a.smithi115.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T07:12:44.920 INFO:journalctl@ceph.mon.a.smithi115.stdout: 2023-10-09T07:12:44.920 INFO:journalctl@ceph.mon.a.smithi115.stdout: ** Compaction Stats [default] ** 2023-10-09T07:12:44.920 INFO:journalctl@ceph.mon.a.smithi115.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T07:12:44.920 INFO:journalctl@ceph.mon.a.smithi115.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T07:12:44.920 INFO:journalctl@ceph.mon.a.smithi115.stdout: L0 2/0 79.73 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 133.0 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T07:12:44.920 INFO:journalctl@ceph.mon.a.smithi115.stdout: Sum 2/0 79.73 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 133.0 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T07:12:44.920 INFO:journalctl@ceph.mon.a.smithi115.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 133.0 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T07:12:44.920 INFO:journalctl@ceph.mon.a.smithi115.stdout: 2023-10-09T07:12:44.921 INFO:journalctl@ceph.mon.a.smithi115.stdout: ** Compaction Stats [default] ** 2023-10-09T07:12:44.921 INFO:journalctl@ceph.mon.a.smithi115.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T07:12:44.921 INFO:journalctl@ceph.mon.a.smithi115.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T07:12:44.921 INFO:journalctl@ceph.mon.a.smithi115.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 133.0 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T07:12:44.921 INFO:journalctl@ceph.mon.a.smithi115.stdout: 2023-10-09T07:12:44.921 INFO:journalctl@ceph.mon.a.smithi115.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T07:12:44.921 INFO:journalctl@ceph.mon.a.smithi115.stdout: 2023-10-09T07:12:44.921 INFO:journalctl@ceph.mon.a.smithi115.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T07:12:44.921 INFO:journalctl@ceph.mon.a.smithi115.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T07:12:44.921 INFO:journalctl@ceph.mon.a.smithi115.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T07:12:44.921 INFO:journalctl@ceph.mon.a.smithi115.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T07:12:44.922 INFO:journalctl@ceph.mon.a.smithi115.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T07:12:44.922 INFO:journalctl@ceph.mon.a.smithi115.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T07:12:44.922 INFO:journalctl@ceph.mon.a.smithi115.stdout: Cumulative compaction: 0.00 GB write, 22.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T07:12:44.922 INFO:journalctl@ceph.mon.a.smithi115.stdout: Interval compaction: 0.00 GB write, 22.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T07:12:44.922 INFO:journalctl@ceph.mon.a.smithi115.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T07:12:44.922 INFO:journalctl@ceph.mon.a.smithi115.stdout: Block cache BinnedLRUCache@0x55a8c14111f0#2 capacity: 512.00 MB usage: 1.19 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2023-10-09T07:12:44.922 INFO:journalctl@ceph.mon.a.smithi115.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.77 KB,0.000146031%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2023-10-09T07:12:44.922 INFO:journalctl@ceph.mon.a.smithi115.stdout: 2023-10-09T07:12:44.922 INFO:journalctl@ceph.mon.a.smithi115.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T07:12:44.922 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: starting mon.a rank 0 at public addrs [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] at bind addrs [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:12:44.923 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: mon.a@-1(???) e1 preinit fsid 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:12:44.923 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: mon.a@-1(???).mds e1 new map 2023-10-09T07:12:44.923 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: mon.a@-1(???).mds e1 print_map 2023-10-09T07:12:44.923 INFO:journalctl@ceph.mon.a.smithi115.stdout: e1 2023-10-09T07:12:44.923 INFO:journalctl@ceph.mon.a.smithi115.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T07:12:44.923 INFO:journalctl@ceph.mon.a.smithi115.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments} 2023-10-09T07:12:44.923 INFO:journalctl@ceph.mon.a.smithi115.stdout: legacy client fscid: -1 2023-10-09T07:12:44.923 INFO:journalctl@ceph.mon.a.smithi115.stdout: 2023-10-09T07:12:44.923 INFO:journalctl@ceph.mon.a.smithi115.stdout: No filesystems configured 2023-10-09T07:12:44.923 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T07:12:44.923 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T07:12:44.923 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T07:12:44.924 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T07:12:44.924 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-09T07:12:44.924 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: expand_channel_meta expand map: {default=false} 2023-10-09T07:12:44.924 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: expand_channel_meta from 'false' to 'false' 2023-10-09T07:12:44.924 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: expand_channel_meta expanded map: {default=false} 2023-10-09T07:12:44.924 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: expand_channel_meta expand map: {default=info} 2023-10-09T07:12:44.924 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: expand_channel_meta from 'info' to 'info' 2023-10-09T07:12:44.924 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: expand_channel_meta expanded map: {default=info} 2023-10-09T07:12:44.924 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: expand_channel_meta expand map: {default=daemon} 2023-10-09T07:12:44.924 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T07:12:44.924 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T07:12:44.925 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: expand_channel_meta expand map: {default=debug} 2023-10-09T07:12:44.925 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T07:12:44.925 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: expand_channel_meta expanded map: {default=debug} 2023-10-09T07:12:44.925 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T07:12:44.925 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: monmap e1: 1 mons at {a=[v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0]} removed_ranks: {} 2023-10-09T07:12:44.925 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: fsmap 2023-10-09T07:12:44.925 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T07:12:44.925 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:44 smithi115 ceph-mon[108061]: mgrmap e1: no daemons active 2023-10-09T07:12:45.233 INFO:teuthology.orchestra.run.smithi115.stdout:Wrote config to /etc/ceph/ceph.conf 2023-10-09T07:12:45.264 INFO:teuthology.orchestra.run.smithi115.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-09T07:12:45.264 INFO:teuthology.orchestra.run.smithi115.stdout:Creating mgr... 2023-10-09T07:12:45.265 INFO:teuthology.orchestra.run.smithi115.stdout:Verifying port 0.0.0.0:9283 ... 2023-10-09T07:12:45.265 INFO:teuthology.orchestra.run.smithi115.stdout:Verifying port 0.0.0.0:8765 ... 2023-10-09T07:12:45.465 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T07:12:45.465 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stderr Failed to reset failed state of unit ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mgr.x.service: Unit ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mgr.x.service not loaded. 2023-10-09T07:12:45.552 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-2817f006-6673-11ee-8db5-212e2dc638e7.target.wants/ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mgr.x.service → /etc/systemd/system/ceph-2817f006-6673-11ee-8db5-212e2dc638e7@.service. 2023-10-09T07:12:45.864 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:45 smithi115 systemd[1]: Starting Ceph mgr.x for 2817f006-6673-11ee-8db5-212e2dc638e7... 2023-10-09T07:12:46.128 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:45 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2372159409' entity='client.admin' 2023-10-09T07:12:46.860 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T07:12:46.860 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout disabled 2023-10-09T07:12:46.868 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T07:12:46.868 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout inactive 2023-10-09T07:12:46.868 INFO:teuthology.orchestra.run.smithi115.stdout:firewalld.service is not enabled 2023-10-09T07:12:46.869 INFO:teuthology.orchestra.run.smithi115.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T07:12:46.876 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T07:12:46.877 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout disabled 2023-10-09T07:12:46.884 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T07:12:46.884 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout inactive 2023-10-09T07:12:46.885 INFO:teuthology.orchestra.run.smithi115.stdout:firewalld.service is not enabled 2023-10-09T07:12:46.885 INFO:teuthology.orchestra.run.smithi115.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2023-10-09T07:12:46.885 INFO:teuthology.orchestra.run.smithi115.stdout:Waiting for mgr to start... 2023-10-09T07:12:46.885 INFO:teuthology.orchestra.run.smithi115.stdout:Waiting for mgr... 2023-10-09T07:12:46.967 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:46 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:46.835+0000 7f978be83200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T07:12:46.967 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:46 smithi115 bash[108270]: 0af3be2fd0302ed2ba00d25513a10f4d84434f2298f42b8233eccf93dae3cf54 2023-10-09T07:12:46.968 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:46 smithi115 systemd[1]: Started Ceph mgr.x for 2817f006-6673-11ee-8db5-212e2dc638e7. 2023-10-09T07:12:47.340 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:47 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:47.013+0000 7f978be83200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T07:12:47.340 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:47 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:47.339+0000 7f978be83200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T07:12:48.117 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:47 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2378985946' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T07:12:48.118 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:47 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:47.855+0000 7f978be83200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T07:12:48.118 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:48 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:48.096+0000 7f978be83200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T07:12:48.192 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 2023-10-09T07:12:48.193 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout { 2023-10-09T07:12:48.193 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "fsid": "2817f006-6673-11ee-8db5-212e2dc638e7", 2023-10-09T07:12:48.193 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T07:12:48.193 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T07:12:48.193 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T07:12:48.193 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T07:12:48.193 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:48.193 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T07:12:48.193 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T07:12:48.193 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 0 2023-10-09T07:12:48.193 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout ], 2023-10-09T07:12:48.194 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T07:12:48.194 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T07:12:48.194 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout ], 2023-10-09T07:12:48.194 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "quorum_age": 3, 2023-10-09T07:12:48.194 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T07:12:48.194 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:48.194 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-10-09T07:12:48.194 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T07:12:48.194 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:48.194 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T07:12:48.194 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:48.194 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T07:12:48.195 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T07:12:48.195 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T07:12:48.195 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T07:12:48.198 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T07:12:48.198 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T07:12:48.198 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:48.198 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T07:12:48.198 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T07:12:48.199 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T07:12:48.199 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T07:12:48.199 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T07:12:48.199 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T07:12:48.199 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T07:12:48.199 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T07:12:48.199 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T07:12:48.199 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:48.199 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T07:12:48.199 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:48.199 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T07:12:48.199 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T07:12:48.199 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:48.200 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T07:12:48.200 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T07:12:48.200 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T07:12:48.200 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T07:12:48.200 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T07:12:48.200 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T07:12:48.200 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T07:12:48.200 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout ], 2023-10-09T07:12:48.200 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T07:12:48.200 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:48.200 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T07:12:48.200 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:48.200 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T07:12:41.167840+0000", 2023-10-09T07:12:48.201 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T07:12:48.201 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:48.201 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T07:12:48.201 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout } 2023-10-09T07:12:48.201 INFO:teuthology.orchestra.run.smithi115.stdout:mgr not available, waiting (1/15)... 2023-10-09T07:12:48.966 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:48 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:48.516+0000 7f978be83200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T07:12:49.466 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:49 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:49.215+0000 7f978be83200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T07:12:49.467 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:49 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:49.300+0000 7f978be83200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T07:12:50.216 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:49 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:49.892+0000 7f978be83200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T07:12:50.217 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:49 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:49.976+0000 7f978be83200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T07:12:50.217 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:50 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:50.064+0000 7f978be83200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T07:12:50.631 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:50 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:50.289+0000 7f978be83200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T07:12:50.631 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:50 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:50.386+0000 7f978be83200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T07:12:50.631 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:50 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:50.483+0000 7f978be83200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T07:12:50.951 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:50 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2118209911' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T07:12:50.951 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:50 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:50.719+0000 7f978be83200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T07:12:50.951 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:50 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:50.816+0000 7f978be83200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T07:12:50.992 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 2023-10-09T07:12:50.992 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout { 2023-10-09T07:12:50.993 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "fsid": "2817f006-6673-11ee-8db5-212e2dc638e7", 2023-10-09T07:12:50.993 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T07:12:50.993 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T07:12:50.993 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T07:12:50.993 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T07:12:50.993 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:50.993 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T07:12:50.993 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T07:12:50.993 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 0 2023-10-09T07:12:50.993 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout ], 2023-10-09T07:12:50.993 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T07:12:50.993 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T07:12:50.994 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout ], 2023-10-09T07:12:50.994 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2023-10-09T07:12:50.994 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T07:12:50.994 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:50.994 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-10-09T07:12:50.994 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T07:12:50.994 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:50.994 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T07:12:50.994 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:50.994 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T07:12:50.994 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T07:12:50.994 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T07:12:50.995 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T07:12:50.995 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T07:12:50.995 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T07:12:50.995 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:50.995 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T07:12:50.995 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T07:12:50.995 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T07:12:50.995 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T07:12:50.995 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T07:12:50.996 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T07:12:50.996 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T07:12:50.996 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T07:12:50.996 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T07:12:50.996 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:50.996 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T07:12:50.996 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:50.996 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T07:12:50.997 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T07:12:50.997 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:50.997 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T07:12:50.997 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T07:12:50.997 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T07:12:50.997 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T07:12:50.997 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T07:12:50.997 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T07:12:50.997 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T07:12:50.998 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout ], 2023-10-09T07:12:50.998 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T07:12:50.998 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:50.998 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T07:12:50.998 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:50.998 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T07:12:41.167840+0000", 2023-10-09T07:12:50.998 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T07:12:50.998 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:50.999 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T07:12:50.999 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout } 2023-10-09T07:12:50.999 INFO:teuthology.orchestra.run.smithi115.stdout:mgr not available, waiting (2/15)... 2023-10-09T07:12:51.216 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:51 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:51.003+0000 7f978be83200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T07:12:51.217 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:51 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:51.090+0000 7f978be83200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T07:12:52.203 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:51 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:51.832+0000 7f978be83200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T07:12:52.203 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:52 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:52.025+0000 7f978be83200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T07:12:52.203 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:52 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:52.202+0000 7f978be83200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T07:12:53.113 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:52 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:52.848+0000 7f978be83200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T07:12:53.113 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:53.036+0000 7f978be83200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T07:12:53.375 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:53.229+0000 7f978be83200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T07:12:53.376 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:53.349+0000 7f978be83200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T07:12:53.658 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/3070778067' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T07:12:53.658 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-mon[108061]: Activating manager daemon x 2023-10-09T07:12:53.658 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-mon[108061]: mgrmap e2: x(active, starting, since 0.0029449s) 2023-10-09T07:12:53.658 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-mon[108061]: from='mgr.14100 172.21.15.115:0/1257887892' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T07:12:53.658 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-mon[108061]: from='mgr.14100 172.21.15.115:0/1257887892' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T07:12:53.658 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-mon[108061]: from='mgr.14100 172.21.15.115:0/1257887892' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T07:12:53.658 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-mon[108061]: from='mgr.14100 172.21.15.115:0/1257887892' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T07:12:53.659 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-mon[108061]: from='mgr.14100 172.21.15.115:0/1257887892' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T07:12:53.659 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-mon[108061]: Manager daemon x is now available 2023-10-09T07:12:53.659 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-mon[108061]: from='mgr.14100 172.21.15.115:0/1257887892' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T07:12:53.659 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-mon[108061]: from='mgr.14100 172.21.15.115:0/1257887892' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T07:12:53.659 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:53 smithi115 ceph-mon[108061]: from='mgr.14100 172.21.15.115:0/1257887892' entity='mgr.x' 2023-10-09T07:12:53.711 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 2023-10-09T07:12:53.711 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout { 2023-10-09T07:12:53.711 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "fsid": "2817f006-6673-11ee-8db5-212e2dc638e7", 2023-10-09T07:12:53.711 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T07:12:53.711 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T07:12:53.711 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T07:12:53.711 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T07:12:53.711 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:53.712 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T07:12:53.712 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T07:12:53.712 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 0 2023-10-09T07:12:53.712 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout ], 2023-10-09T07:12:53.712 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T07:12:53.712 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T07:12:53.712 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout ], 2023-10-09T07:12:53.712 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2023-10-09T07:12:53.712 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T07:12:53.712 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:53.712 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-10-09T07:12:53.712 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T07:12:53.713 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:53.713 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T07:12:53.713 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:53.713 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T07:12:53.713 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T07:12:53.713 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T07:12:53.713 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T07:12:53.713 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T07:12:53.713 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T07:12:53.713 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:53.713 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T07:12:53.713 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T07:12:53.714 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T07:12:53.714 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T07:12:53.714 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T07:12:53.714 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T07:12:53.714 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T07:12:53.714 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T07:12:53.714 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T07:12:53.714 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:53.714 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T07:12:53.714 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:53.714 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T07:12:53.715 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T07:12:53.715 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:53.715 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T07:12:53.715 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T07:12:53.715 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T07:12:53.715 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T07:12:53.715 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T07:12:53.715 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T07:12:53.715 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T07:12:53.715 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout ], 2023-10-09T07:12:53.716 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T07:12:53.716 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:53.716 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T07:12:53.716 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:53.716 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T07:12:41.167840+0000", 2023-10-09T07:12:53.716 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T07:12:53.716 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:53.716 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T07:12:53.716 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout } 2023-10-09T07:12:53.716 INFO:teuthology.orchestra.run.smithi115.stdout:mgr not available, waiting (3/15)... 2023-10-09T07:12:54.716 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:54 smithi115 ceph-mon[108061]: from='mgr.14100 172.21.15.115:0/1257887892' entity='mgr.x' 2023-10-09T07:12:54.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:54 smithi115 ceph-mon[108061]: from='mgr.14100 172.21.15.115:0/1257887892' entity='mgr.x' 2023-10-09T07:12:54.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:54 smithi115 ceph-mon[108061]: mgrmap e3: x(active, since 1.00756s) 2023-10-09T07:12:56.599 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:56 smithi115 ceph-mon[108061]: mgrmap e4: x(active, since 2s) 2023-10-09T07:12:56.599 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:56 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/3331523723' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T07:12:56.669 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 2023-10-09T07:12:56.669 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout { 2023-10-09T07:12:56.669 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "fsid": "2817f006-6673-11ee-8db5-212e2dc638e7", 2023-10-09T07:12:56.669 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T07:12:56.669 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T07:12:56.669 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T07:12:56.669 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T07:12:56.669 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:56.669 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T07:12:56.669 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T07:12:56.670 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 0 2023-10-09T07:12:56.670 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout ], 2023-10-09T07:12:56.670 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T07:12:56.670 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T07:12:56.670 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout ], 2023-10-09T07:12:56.670 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "quorum_age": 11, 2023-10-09T07:12:56.670 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T07:12:56.670 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:56.670 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-10-09T07:12:56.670 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T07:12:56.670 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:56.670 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T07:12:56.670 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:56.671 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T07:12:56.671 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T07:12:56.671 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T07:12:56.671 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T07:12:56.671 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T07:12:56.671 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T07:12:56.671 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:56.671 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T07:12:56.671 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T07:12:56.671 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T07:12:56.671 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T07:12:56.671 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T07:12:56.672 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T07:12:56.672 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T07:12:56.672 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T07:12:56.672 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T07:12:56.672 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:56.672 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T07:12:56.672 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:56.672 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T07:12:56.672 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T07:12:56.672 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:56.672 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T07:12:56.672 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T07:12:56.672 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout ], 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T07:12:41.167840+0000", 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout }, 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T07:12:56.673 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout } 2023-10-09T07:12:56.674 INFO:teuthology.orchestra.run.smithi115.stdout:mgr is available 2023-10-09T07:12:57.481 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 2023-10-09T07:12:57.481 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T07:12:57.482 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout fsid = 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:12:57.482 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T07:12:57.482 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T07:12:57.482 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T07:12:57.482 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T07:12:57.482 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 2023-10-09T07:12:57.482 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T07:12:57.482 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T07:12:57.482 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 2023-10-09T07:12:57.482 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T07:12:57.482 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T07:12:57.483 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T07:12:57.483 INFO:teuthology.orchestra.run.smithi115.stdout:Enabling cephadm module... 2023-10-09T07:12:57.716 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:57 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2567631504' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T07:12:58.889 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:58 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/3994602267' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T07:12:58.900 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:58 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: ignoring --setuser ceph since I am not root 2023-10-09T07:12:58.900 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:58 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: ignoring --setgroup ceph since I am not root 2023-10-09T07:12:58.900 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:58 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:58.723+0000 7f7c1c134200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T07:12:58.900 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:58 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:58.889+0000 7f7c1c134200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T07:12:59.466 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:59 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:59.213+0000 7f7c1c134200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T07:12:59.871 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:59 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:59.706+0000 7f7c1c134200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T07:12:59.872 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:59 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/3994602267' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T07:12:59.872 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:12:59 smithi115 ceph-mon[108061]: mgrmap e5: x(active, since 5s) 2023-10-09T07:13:00.216 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:12:59 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:12:59.937+0000 7f7c1c134200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T07:13:00.667 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:00 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:00.369+0000 7f7c1c134200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T07:13:01.466 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:01 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:01.101+0000 7f7c1c134200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T07:13:01.467 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:01 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:01.188+0000 7f7c1c134200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T07:13:02.203 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:01 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/3463928618' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T07:13:02.204 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:01 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:01.797+0000 7f7c1c134200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T07:13:02.204 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:01 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:01.884+0000 7f7c1c134200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T07:13:02.204 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:01 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:01.976+0000 7f7c1c134200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T07:13:02.466 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:02 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:02.202+0000 7f7c1c134200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T07:13:02.467 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:02 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:02.295+0000 7f7c1c134200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T07:13:02.467 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:02 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:02.392+0000 7f7c1c134200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T07:13:02.888 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:02 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:02.606+0000 7f7c1c134200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T07:13:02.888 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:02 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:02.703+0000 7f7c1c134200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T07:13:02.888 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:02 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:02.887+0000 7f7c1c134200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T07:13:03.216 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:02 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:02.973+0000 7f7c1c134200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T07:13:03.919 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout { 2023-10-09T07:13:03.919 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-10-09T07:13:03.919 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T07:13:03.919 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "active_name": "x", 2023-10-09T07:13:03.919 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T07:13:03.919 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout } 2023-10-09T07:13:03.920 INFO:teuthology.orchestra.run.smithi115.stdout:Waiting for the mgr to restart... 2023-10-09T07:13:03.920 INFO:teuthology.orchestra.run.smithi115.stdout:Waiting for mgr epoch 5... 2023-10-09T07:13:04.107 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:03 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:03.724+0000 7f7c1c134200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T07:13:04.107 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:03 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:03.920+0000 7f7c1c134200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T07:13:04.107 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:04 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:04.106+0000 7f7c1c134200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T07:13:05.136 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:04 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:04.770+0000 7f7c1c134200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T07:13:05.136 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:04 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:04.952+0000 7f7c1c134200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T07:13:05.136 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:05.135+0000 7f7c1c134200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T07:13:05.466 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:05.251+0000 7f7c1c134200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T07:13:05.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: Active manager daemon x restarted 2023-10-09T07:13:05.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: Activating manager daemon x 2023-10-09T07:13:05.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: osdmap e2: 0 total, 0 up, 0 in 2023-10-09T07:13:05.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: mgrmap e6: x(active, starting, since 0.00319915s) 2023-10-09T07:13:05.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T07:13:05.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T07:13:05.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T07:13:05.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T07:13:05.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T07:13:05.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: Manager daemon x is now available 2023-10-09T07:13:05.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:05.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:05.969 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:13:05.969 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:13:05.969 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:13:05.969 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T07:13:05.969 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:05 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T07:13:06.571 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:06 smithi115 ceph-mon[108061]: Found migration_current of "None". Setting to last migration. 2023-10-09T07:13:06.572 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:06 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:06.572 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:06 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:06.572 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:06 smithi115 ceph-mon[108061]: mgrmap e7: x(active, since 1.00728s) 2023-10-09T07:13:06.645 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout { 2023-10-09T07:13:06.645 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-10-09T07:13:06.645 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T07:13:06.645 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout } 2023-10-09T07:13:06.646 INFO:teuthology.orchestra.run.smithi115.stdout:mgr epoch 5 is available 2023-10-09T07:13:06.646 INFO:teuthology.orchestra.run.smithi115.stdout:Setting orchestrator backend to cephadm... 2023-10-09T07:13:07.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:07 smithi115 ceph-mon[108061]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T07:13:07.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:07 smithi115 ceph-mon[108061]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T07:13:07.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:07 smithi115 ceph-mon[108061]: [09/Oct/2023:07:13:06] ENGINE Bus STARTING 2023-10-09T07:13:07.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:07 smithi115 ceph-mon[108061]: [09/Oct/2023:07:13:06] ENGINE Serving on http://172.21.15.115:8765 2023-10-09T07:13:07.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:07 smithi115 ceph-mon[108061]: [09/Oct/2023:07:13:06] ENGINE Serving on https://172.21.15.115:7150 2023-10-09T07:13:07.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:07 smithi115 ceph-mon[108061]: [09/Oct/2023:07:13:06] ENGINE Bus STARTED 2023-10-09T07:13:07.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:07 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:13:07.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:07 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:07.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:07 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:13:08.361 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout value unchanged 2023-10-09T07:13:08.361 INFO:teuthology.orchestra.run.smithi115.stdout:Generating ssh key... 2023-10-09T07:13:08.716 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:08 smithi115 ceph-mon[108061]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:13:08.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:08 smithi115 ceph-mon[108061]: mgrmap e8: x(active, since 2s) 2023-10-09T07:13:09.463 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: Generating public/private rsa key pair. 2023-10-09T07:13:09.463 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: Your identification has been saved in /tmp/tmpr46hntle/key. 2023-10-09T07:13:09.463 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: Your public key has been saved in /tmp/tmpr46hntle/key.pub. 2023-10-09T07:13:09.463 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: The key fingerprint is: 2023-10-09T07:13:09.463 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: SHA256:kTn0BId++By2uOC+IC4fhhWHDU6u1MHUM2ypNgte7aw ceph-2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:13:09.463 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: The key's randomart image is: 2023-10-09T07:13:09.463 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: +---[RSA 3072]----+ 2023-10-09T07:13:09.464 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: | =oo . ooo | 2023-10-09T07:13:09.464 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: | +.=.B ..* | 2023-10-09T07:13:09.464 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: | .=.* o.=.. | 2023-10-09T07:13:09.464 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: |.o B . oo+ | 2023-10-09T07:13:09.464 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: |o = = S* o | 2023-10-09T07:13:09.464 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: | + . o. . + | 2023-10-09T07:13:09.464 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: |. + o. . . | 2023-10-09T07:13:09.464 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: |.o E .. . | 2023-10-09T07:13:09.465 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: | oo .o. | 2023-10-09T07:13:09.465 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: +----[SHA256]-----+ 2023-10-09T07:13:09.716 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-mon[108061]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:13:09.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:09.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:09 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:10.370 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2rlQtPdW8Ss58xYNWklI7qzp9fFRqk9XRWSPA9RJ4XYsz8RyiRILMrcZQeaKRTDX0m0jMCL755UaYXFcmEb6MmNoG8OqOgmuIMEhmtOcVYgRNX9rs6+XvfFXp2gjqc0wrMcF44DjfR82C+c10v7F5mbqqcewyhqj/7di2kaGKfVp7nwaEABrvQwGtZKpASxg2dLlkSX8BGHK4Hr+bElED0mWaJLc+cXN8zCRGUmljUyWSpBU+E+jxZm+0JgKq8AWdjt1LWZJWX7fWHnTVHfj7+JqaqV84mlBPrhXbgP6+xwxZqkwU+V1QtMo45PyAky6bRitrz3M6E0FF9fAbHG9vgZ47V1r55u2atuzYqoW8ozjGCYbzh4BTydjyU/vOtjo5K+EU2CBBfWF8iJFhO1mzytseBwQg7Br6dug7kVzXnmxfKZ3i3sKFDpTcCp/BV2yJyJO9BGXZ087x8SsGx4E/Sv0nWg9h/AE11qU3XLaOd+cxrhVQd1hvxMLkQnARc3U= ceph-2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:13:10.370 INFO:teuthology.orchestra.run.smithi115.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-09T07:13:10.370 INFO:teuthology.orchestra.run.smithi115.stdout:Adding key to root@localhost authorized_keys... 2023-10-09T07:13:10.370 INFO:teuthology.orchestra.run.smithi115.stdout:Adding host smithi115... 2023-10-09T07:13:10.716 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:10 smithi115 ceph-mon[108061]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:13:10.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:10 smithi115 ceph-mon[108061]: Generating ssh key... 2023-10-09T07:13:11.675 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:11 smithi115 ceph-mon[108061]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:13:12.795 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:12 smithi115 ceph-mon[108061]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi115", "addr": "172.21.15.115", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:13:12.795 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:12 smithi115 ceph-mon[108061]: Deploying cephadm binary to smithi115 2023-10-09T07:13:13.384 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout Added host 'smithi115' with addr '172.21.15.115' 2023-10-09T07:13:13.385 INFO:teuthology.orchestra.run.smithi115.stdout:Deploying unmanaged mon service... 2023-10-09T07:13:14.200 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:14 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:14.200 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:14 smithi115 ceph-mon[108061]: Added host smithi115 2023-10-09T07:13:14.200 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:14 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:13:14.201 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:14 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:14.476 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-10-09T07:13:14.476 INFO:teuthology.orchestra.run.smithi115.stdout:Deploying unmanaged mgr service... 2023-10-09T07:13:15.318 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-10-09T07:13:15.885 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:15 smithi115 ceph-mon[108061]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:13:15.886 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:15 smithi115 ceph-mon[108061]: Saving service mon spec with placement count:5 2023-10-09T07:13:15.886 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:15 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:15.886 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:15 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:15.886 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:15 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:16.854 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:16 smithi115 ceph-mon[108061]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:13:16.854 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:16 smithi115 ceph-mon[108061]: Saving service mgr spec with placement count:2 2023-10-09T07:13:16.854 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:16 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/3420340545' entity='client.admin' 2023-10-09T07:13:17.700 INFO:teuthology.orchestra.run.smithi115.stdout:Enabling the dashboard module... 2023-10-09T07:13:18.088 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:17 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:18.088 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:17 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/3505263470' entity='client.admin' 2023-10-09T07:13:18.088 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:17 smithi115 ceph-mon[108061]: from='mgr.14120 172.21.15.115:0/2638144050' entity='mgr.x' 2023-10-09T07:13:19.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:18 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2499215167' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T07:13:19.217 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:19 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: ignoring --setuser ceph since I am not root 2023-10-09T07:13:19.217 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:19 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: ignoring --setgroup ceph since I am not root 2023-10-09T07:13:19.217 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:19 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:19.137+0000 7f2d288dd200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T07:13:19.470 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:19 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:19.306+0000 7f2d288dd200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T07:13:19.924 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:19 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:19.657+0000 7f2d288dd200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T07:13:20.202 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:20 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:20.163+0000 7f2d288dd200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T07:13:20.202 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:19 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2499215167' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T07:13:20.203 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:19 smithi115 ceph-mon[108061]: mgrmap e9: x(active, since 13s) 2023-10-09T07:13:20.203 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:19 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/313181111' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T07:13:20.259 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout { 2023-10-09T07:13:20.259 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-10-09T07:13:20.260 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T07:13:20.260 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "active_name": "x", 2023-10-09T07:13:20.260 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T07:13:20.260 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout } 2023-10-09T07:13:20.260 INFO:teuthology.orchestra.run.smithi115.stdout:Waiting for the mgr to restart... 2023-10-09T07:13:20.260 INFO:teuthology.orchestra.run.smithi115.stdout:Waiting for mgr epoch 9... 2023-10-09T07:13:20.466 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:20 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:20.411+0000 7f2d288dd200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T07:13:21.216 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:20 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:20.838+0000 7f2d288dd200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T07:13:21.966 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:21 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:21.535+0000 7f2d288dd200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T07:13:21.967 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:21 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:21.622+0000 7f2d288dd200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T07:13:22.614 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:22 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:22.218+0000 7f2d288dd200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T07:13:22.614 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:22 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:22.303+0000 7f2d288dd200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T07:13:22.614 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:22 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:22.391+0000 7f2d288dd200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T07:13:22.966 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:22 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:22.613+0000 7f2d288dd200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T07:13:22.967 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:22 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:22.707+0000 7f2d288dd200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T07:13:22.967 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:22 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:22.801+0000 7f2d288dd200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T07:13:23.295 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:23 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:23.015+0000 7f2d288dd200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T07:13:23.295 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:23 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:23.111+0000 7f2d288dd200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T07:13:23.716 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:23 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:23.294+0000 7f2d288dd200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T07:13:23.717 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:23 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:23.381+0000 7f2d288dd200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T07:13:24.466 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:24 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:24.128+0000 7f2d288dd200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T07:13:24.467 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:24 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:24.322+0000 7f2d288dd200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T07:13:24.966 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:24 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:24.501+0000 7f2d288dd200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T07:13:25.466 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:25.150+0000 7f2d288dd200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T07:13:25.467 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:25.332+0000 7f2d288dd200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T07:13:25.967 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:25.515+0000 7f2d288dd200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T07:13:25.967 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:13:25.631+0000 7f2d288dd200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T07:13:25.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-mon[108061]: Active manager daemon x restarted 2023-10-09T07:13:25.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-mon[108061]: Activating manager daemon x 2023-10-09T07:13:25.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-mon[108061]: osdmap e3: 0 total, 0 up, 0 in 2023-10-09T07:13:25.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-mon[108061]: mgrmap e10: x(active, starting, since 0.00348676s) 2023-10-09T07:13:25.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T07:13:25.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T07:13:25.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T07:13:25.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T07:13:25.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T07:13:25.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-mon[108061]: Manager daemon x is now available 2023-10-09T07:13:25.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:13:25.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:25 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:13:26.959 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:26 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T07:13:26.959 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:26 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T07:13:26.959 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:26 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:26.959 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:26 smithi115 ceph-mon[108061]: mgrmap e11: x(active, since 1.00649s) 2023-10-09T07:13:27.032 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout { 2023-10-09T07:13:27.033 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-10-09T07:13:27.033 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T07:13:27.033 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout } 2023-10-09T07:13:27.033 INFO:teuthology.orchestra.run.smithi115.stdout:mgr epoch 9 is available 2023-10-09T07:13:27.033 INFO:teuthology.orchestra.run.smithi115.stdout:Generating a dashboard self-signed certificate... 2023-10-09T07:13:27.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:27 smithi115 ceph-mon[108061]: [09/Oct/2023:07:13:26] ENGINE Bus STARTING 2023-10-09T07:13:27.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:27 smithi115 ceph-mon[108061]: [09/Oct/2023:07:13:26] ENGINE Serving on https://172.21.15.115:7150 2023-10-09T07:13:27.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:27 smithi115 ceph-mon[108061]: [09/Oct/2023:07:13:26] ENGINE Serving on http://172.21.15.115:8765 2023-10-09T07:13:27.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:27 smithi115 ceph-mon[108061]: [09/Oct/2023:07:13:26] ENGINE Bus STARTED 2023-10-09T07:13:27.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:27 smithi115 ceph-mon[108061]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T07:13:27.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:27 smithi115 ceph-mon[108061]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T07:13:27.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:27 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:27.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:27 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:27.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:27 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:28.365 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-10-09T07:13:28.365 INFO:teuthology.orchestra.run.smithi115.stdout:Creating initial admin user... 2023-10-09T07:13:28.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:28 smithi115 ceph-mon[108061]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:13:28.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:28 smithi115 ceph-mon[108061]: mgrmap e12: x(active, since 2s) 2023-10-09T07:13:29.431 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$zDc0A5FjbVdiGTKD9hdJY.XPxSD9C41OwipsIlA1dKbDPKcJoJkJC", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696835609, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-09T07:13:29.431 INFO:teuthology.orchestra.run.smithi115.stdout:Fetching dashboard port number... 2023-10-09T07:13:29.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:29 smithi115 ceph-mon[108061]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:13:29.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:29 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:30.273 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stdout 8443 2023-10-09T07:13:30.282 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T07:13:30.282 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout disabled 2023-10-09T07:13:30.289 INFO:teuthology.orchestra.run.smithi115.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T07:13:30.290 INFO:teuthology.orchestra.run.smithi115.stdout:systemctl: stdout inactive 2023-10-09T07:13:30.290 INFO:teuthology.orchestra.run.smithi115.stdout:firewalld.service is not enabled 2023-10-09T07:13:30.290 INFO:teuthology.orchestra.run.smithi115.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-09T07:13:30.291 INFO:teuthology.orchestra.run.smithi115.stdout:Ceph Dashboard is now available at: 2023-10-09T07:13:30.292 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:13:30.292 INFO:teuthology.orchestra.run.smithi115.stdout: URL: https://smithi115.front.sepia.ceph.com:8443/ 2023-10-09T07:13:30.292 INFO:teuthology.orchestra.run.smithi115.stdout: User: admin 2023-10-09T07:13:30.292 INFO:teuthology.orchestra.run.smithi115.stdout: Password: 4l6knild7w 2023-10-09T07:13:30.292 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:13:30.292 INFO:teuthology.orchestra.run.smithi115.stdout:Saving cluster configuration to /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/config directory 2023-10-09T07:13:30.293 INFO:teuthology.orchestra.run.smithi115.stdout:Enabling autotune for osd_memory_target 2023-10-09T07:13:30.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:30 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/1112234024' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T07:13:32.823 INFO:teuthology.orchestra.run.smithi115.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-10-09T07:13:32.823 INFO:teuthology.orchestra.run.smithi115.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-09T07:13:32.823 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:13:32.823 INFO:teuthology.orchestra.run.smithi115.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-09T07:13:32.823 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:13:32.824 INFO:teuthology.orchestra.run.smithi115.stdout:Or, if you are only running a single cluster on this host: 2023-10-09T07:13:32.824 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:13:32.824 INFO:teuthology.orchestra.run.smithi115.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-09T07:13:32.824 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:13:32.824 INFO:teuthology.orchestra.run.smithi115.stdout:Please consider enabling telemetry to help improve Ceph: 2023-10-09T07:13:32.824 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:13:32.824 INFO:teuthology.orchestra.run.smithi115.stdout: ceph telemetry on 2023-10-09T07:13:32.824 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:13:32.824 INFO:teuthology.orchestra.run.smithi115.stdout:For more information see: 2023-10-09T07:13:32.825 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:13:32.825 INFO:teuthology.orchestra.run.smithi115.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2023-10-09T07:13:32.825 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:13:32.825 INFO:teuthology.orchestra.run.smithi115.stdout:Bootstrap complete. 2023-10-09T07:13:32.865 INFO:tasks.cephadm:Fetching config... 2023-10-09T07:13:32.865 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2023-10-09T07:13:32.865 DEBUG:teuthology.orchestra.run.smithi115:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-09T07:13:32.885 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-09T07:13:32.885 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2023-10-09T07:13:32.885 DEBUG:teuthology.orchestra.run.smithi115:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-09T07:13:32.943 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-09T07:13:32.944 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2023-10-09T07:13:32.944 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd if=/var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-09T07:13:32.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:32 smithi115 ceph-mon[108061]: mgrmap e13: x(active, since 6s) 2023-10-09T07:13:32.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:32 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2766966103' entity='client.admin' 2023-10-09T07:13:32.994 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-09T07:13:32.995 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2023-10-09T07:13:32.995 DEBUG:teuthology.orchestra.run.smithi115:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-09T07:13:33.053 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-09T07:13:33.053 DEBUG:teuthology.orchestra.run.smithi115:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2rlQtPdW8Ss58xYNWklI7qzp9fFRqk9XRWSPA9RJ4XYsz8RyiRILMrcZQeaKRTDX0m0jMCL755UaYXFcmEb6MmNoG8OqOgmuIMEhmtOcVYgRNX9rs6+XvfFXp2gjqc0wrMcF44DjfR82C+c10v7F5mbqqcewyhqj/7di2kaGKfVp7nwaEABrvQwGtZKpASxg2dLlkSX8BGHK4Hr+bElED0mWaJLc+cXN8zCRGUmljUyWSpBU+E+jxZm+0JgKq8AWdjt1LWZJWX7fWHnTVHfj7+JqaqV84mlBPrhXbgP6+xwxZqkwU+V1QtMo45PyAky6bRitrz3M6E0FF9fAbHG9vgZ47V1r55u2atuzYqoW8ozjGCYbzh4BTydjyU/vOtjo5K+EU2CBBfWF8iJFhO1mzytseBwQg7Br6dug7kVzXnmxfKZ3i3sKFDpTcCp/BV2yJyJO9BGXZ087x8SsGx4E/Sv0nWg9h/AE11qU3XLaOd+cxrhVQd1hvxMLkQnARc3U= ceph-2817f006-6673-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T07:13:33.132 INFO:teuthology.orchestra.run.smithi115.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2rlQtPdW8Ss58xYNWklI7qzp9fFRqk9XRWSPA9RJ4XYsz8RyiRILMrcZQeaKRTDX0m0jMCL755UaYXFcmEb6MmNoG8OqOgmuIMEhmtOcVYgRNX9rs6+XvfFXp2gjqc0wrMcF44DjfR82C+c10v7F5mbqqcewyhqj/7di2kaGKfVp7nwaEABrvQwGtZKpASxg2dLlkSX8BGHK4Hr+bElED0mWaJLc+cXN8zCRGUmljUyWSpBU+E+jxZm+0JgKq8AWdjt1LWZJWX7fWHnTVHfj7+JqaqV84mlBPrhXbgP6+xwxZqkwU+V1QtMo45PyAky6bRitrz3M6E0FF9fAbHG9vgZ47V1r55u2atuzYqoW8ozjGCYbzh4BTydjyU/vOtjo5K+EU2CBBfWF8iJFhO1mzytseBwQg7Br6dug7kVzXnmxfKZ3i3sKFDpTcCp/BV2yJyJO9BGXZ087x8SsGx4E/Sv0nWg9h/AE11qU3XLaOd+cxrhVQd1hvxMLkQnARc3U= ceph-2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:13:33.143 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-09T07:13:33.400 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:13:34.984 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-09T07:13:34.984 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-09T07:13:35.361 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:13:35.841 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:35 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2655459421' entity='client.admin' 2023-10-09T07:13:35.841 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:35 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:35.841 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:35 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:35.841 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:35 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi115", "name": "osd_memory_target"}]: dispatch 2023-10-09T07:13:35.841 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:35 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:35.841 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:35 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:13:35.841 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:35 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:36.819 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-09T07:13:36.819 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-09T07:13:37.057 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:13:37.716 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:37 smithi115 ceph-mon[108061]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:13:37.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:37 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:37.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:37 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:13:37.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:37 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:13:37.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:37 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T07:13:37.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:37 smithi115 ceph-mon[108061]: Updating smithi115:/etc/ceph/ceph.conf 2023-10-09T07:13:37.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:37 smithi115 ceph-mon[108061]: Updating smithi115:/var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T07:13:37.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:37 smithi115 ceph-mon[108061]: Updating smithi115:/etc/ceph/ceph.client.admin.keyring 2023-10-09T07:13:38.530 INFO:teuthology.orchestra.run.smithi115.stderr:adjusted tunables profile to default 2023-10-09T07:13:38.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:38 smithi115 ceph-mon[108061]: Updating smithi115:/var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T07:13:38.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:38 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:38.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:38 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:38.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:38 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:38.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:38 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/609339406' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T07:13:39.869 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:39 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/609339406' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T07:13:39.869 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:39 smithi115 ceph-mon[108061]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T07:13:40.056 INFO:tasks.cephadm:Adding mon.a on smithi115 2023-10-09T07:13:40.056 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '1;smithi115:172.21.15.115=a' 2023-10-09T07:13:40.278 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:13:41.401 INFO:teuthology.orchestra.run.smithi115.stdout:Scheduled mon update... 2023-10-09T07:13:41.815 INFO:tasks.cephadm:Waiting for 1 mons in monmap... 2023-10-09T07:13:41.816 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T07:13:42.379 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:13:42.631 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "1;smithi115:172.21.15.115=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:13:42.631 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: Saving service mon spec with placement smithi115:172.21.15.115=a;count:1 2023-10-09T07:13:42.631 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:42.632 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:13:42.632 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:13:42.632 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T07:13:42.632 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:42.632 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:42.632 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:42.632 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:42.633 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:42.633 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:42.633 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T07:13:42.633 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T07:13:42.633 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T07:13:42.633 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:13:42.633 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:42 smithi115 ceph-mon[108061]: Reconfiguring daemon mon.a on smithi115 2023-10-09T07:13:43.583 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:13:43.583 INFO:teuthology.orchestra.run.smithi115.stdout:{"epoch":1,"fsid":"2817f006-6673-11ee-8db5-212e2dc638e7","modified":"2023-10-09T07:12:39.278017Z","created":"2023-10-09T07:12:39.278017Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:3300","nonce":0},{"type":"v1","addr":"172.21.15.115:6789","nonce":0}]},"addr":"172.21.15.115:6789/0","public_addr":"172.21.15.115:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-09T07:13:43.585 INFO:teuthology.orchestra.run.smithi115.stderr:dumped monmap epoch 1 2023-10-09T07:13:43.813 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:43 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:43.813 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:43 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:43.965 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-09T07:13:43.965 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-09T07:13:44.178 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:13:44.657 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:44 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/1671499201' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T07:13:45.277 INFO:teuthology.orchestra.run.smithi115.stdout:# minimal ceph.conf for 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:13:45.277 INFO:teuthology.orchestra.run.smithi115.stdout:[global] 2023-10-09T07:13:45.277 INFO:teuthology.orchestra.run.smithi115.stdout: fsid = 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:13:45.277 INFO:teuthology.orchestra.run.smithi115.stdout: mon_host = [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] 2023-10-09T07:13:45.646 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:45 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2650176943' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:13:45.648 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-09T07:13:45.648 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2023-10-09T07:13:45.648 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T07:13:45.680 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2023-10-09T07:13:45.680 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T07:13:45.747 INFO:tasks.cephadm:Adding mgr.x on smithi115 2023-10-09T07:13:45.748 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '1;smithi115=x' 2023-10-09T07:13:45.999 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:13:46.724 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:46 smithi115 ceph-mon[108061]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T07:13:47.116 INFO:teuthology.orchestra.run.smithi115.stdout:Scheduled mgr update... 2023-10-09T07:13:47.480 INFO:tasks.cephadm:Deploying OSDs... 2023-10-09T07:13:47.480 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2023-10-09T07:13:47.481 DEBUG:teuthology.orchestra.run.smithi115:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T07:13:47.499 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T07:13:47.499 DEBUG:teuthology.orchestra.run.smithi115:> stat /dev/vg_nvme/lv_1 2023-10-09T07:13:47.558 INFO:teuthology.orchestra.run.smithi115.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T07:13:47.559 INFO:teuthology.orchestra.run.smithi115.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T07:13:47.559 INFO:teuthology.orchestra.run.smithi115.stdout:Device: 6h/6d Inode: 395097 Links: 1 2023-10-09T07:13:47.559 INFO:teuthology.orchestra.run.smithi115.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T07:13:47.559 INFO:teuthology.orchestra.run.smithi115.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T07:13:47.559 INFO:teuthology.orchestra.run.smithi115.stdout:Access: 2023-10-09 07:13:34.829632093 +0000 2023-10-09T07:13:47.559 INFO:teuthology.orchestra.run.smithi115.stdout:Modify: 2023-10-09 07:11:11.315681007 +0000 2023-10-09T07:13:47.559 INFO:teuthology.orchestra.run.smithi115.stdout:Change: 2023-10-09 07:11:11.315681007 +0000 2023-10-09T07:13:47.559 INFO:teuthology.orchestra.run.smithi115.stdout: Birth: - 2023-10-09T07:13:47.559 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T07:13:47.627 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records in 2023-10-09T07:13:47.627 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records out 2023-10-09T07:13:47.627 INFO:teuthology.orchestra.run.smithi115.stderr:512 bytes copied, 0.000225711 s, 2.3 MB/s 2023-10-09T07:13:47.628 DEBUG:teuthology.orchestra.run.smithi115:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T07:13:47.687 DEBUG:teuthology.orchestra.run.smithi115:> stat /dev/vg_nvme/lv_2 2023-10-09T07:13:47.747 INFO:teuthology.orchestra.run.smithi115.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T07:13:47.748 INFO:teuthology.orchestra.run.smithi115.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T07:13:47.748 INFO:teuthology.orchestra.run.smithi115.stdout:Device: 6h/6d Inode: 395199 Links: 1 2023-10-09T07:13:47.748 INFO:teuthology.orchestra.run.smithi115.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T07:13:47.748 INFO:teuthology.orchestra.run.smithi115.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T07:13:47.748 INFO:teuthology.orchestra.run.smithi115.stdout:Access: 2023-10-09 07:13:34.830632075 +0000 2023-10-09T07:13:47.748 INFO:teuthology.orchestra.run.smithi115.stdout:Modify: 2023-10-09 07:11:11.828671530 +0000 2023-10-09T07:13:47.749 INFO:teuthology.orchestra.run.smithi115.stdout:Change: 2023-10-09 07:11:11.828671530 +0000 2023-10-09T07:13:47.749 INFO:teuthology.orchestra.run.smithi115.stdout: Birth: - 2023-10-09T07:13:47.749 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T07:13:47.815 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records in 2023-10-09T07:13:47.815 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records out 2023-10-09T07:13:47.815 INFO:teuthology.orchestra.run.smithi115.stderr:512 bytes copied, 0.00021661 s, 2.4 MB/s 2023-10-09T07:13:47.816 DEBUG:teuthology.orchestra.run.smithi115:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T07:13:47.875 DEBUG:teuthology.orchestra.run.smithi115:> stat /dev/vg_nvme/lv_3 2023-10-09T07:13:47.934 INFO:teuthology.orchestra.run.smithi115.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T07:13:47.966 INFO:teuthology.orchestra.run.smithi115.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T07:13:47.966 INFO:teuthology.orchestra.run.smithi115.stdout:Device: 6h/6d Inode: 400465 Links: 1 2023-10-09T07:13:47.966 INFO:teuthology.orchestra.run.smithi115.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T07:13:47.966 INFO:teuthology.orchestra.run.smithi115.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T07:13:47.966 INFO:teuthology.orchestra.run.smithi115.stdout:Access: 2023-10-09 07:13:34.830632075 +0000 2023-10-09T07:13:47.966 INFO:teuthology.orchestra.run.smithi115.stdout:Modify: 2023-10-09 07:11:12.343662017 +0000 2023-10-09T07:13:47.967 INFO:teuthology.orchestra.run.smithi115.stdout:Change: 2023-10-09 07:11:12.343662017 +0000 2023-10-09T07:13:47.967 INFO:teuthology.orchestra.run.smithi115.stdout: Birth: - 2023-10-09T07:13:47.967 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T07:13:48.003 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records in 2023-10-09T07:13:48.004 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records out 2023-10-09T07:13:48.004 INFO:teuthology.orchestra.run.smithi115.stderr:512 bytes copied, 0.000223187 s, 2.3 MB/s 2023-10-09T07:13:48.005 DEBUG:teuthology.orchestra.run.smithi115:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T07:13:48.064 DEBUG:teuthology.orchestra.run.smithi115:> stat /dev/vg_nvme/lv_4 2023-10-09T07:13:48.123 INFO:teuthology.orchestra.run.smithi115.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T07:13:48.123 INFO:teuthology.orchestra.run.smithi115.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T07:13:48.124 INFO:teuthology.orchestra.run.smithi115.stdout:Device: 6h/6d Inode: 399472 Links: 1 2023-10-09T07:13:48.124 INFO:teuthology.orchestra.run.smithi115.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T07:13:48.124 INFO:teuthology.orchestra.run.smithi115.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T07:13:48.124 INFO:teuthology.orchestra.run.smithi115.stdout:Access: 2023-10-09 07:13:34.830632075 +0000 2023-10-09T07:13:48.124 INFO:teuthology.orchestra.run.smithi115.stdout:Modify: 2023-10-09 07:11:12.861652449 +0000 2023-10-09T07:13:48.124 INFO:teuthology.orchestra.run.smithi115.stdout:Change: 2023-10-09 07:11:12.861652449 +0000 2023-10-09T07:13:48.124 INFO:teuthology.orchestra.run.smithi115.stdout: Birth: - 2023-10-09T07:13:48.124 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T07:13:48.191 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records in 2023-10-09T07:13:48.191 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records out 2023-10-09T07:13:48.191 INFO:teuthology.orchestra.run.smithi115.stderr:512 bytes copied, 0.000234065 s, 2.2 MB/s 2023-10-09T07:13:48.192 DEBUG:teuthology.orchestra.run.smithi115:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T07:13:48.251 INFO:tasks.cephadm:Deploying osd.0 on smithi115 with /dev/vg_nvme/lv_4... 2023-10-09T07:13:48.251 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T07:13:48.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "1;smithi115=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:13:48.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: Saving service mgr spec with placement smithi115=x;count:1 2023-10-09T07:13:48.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:48.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:13:48.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:13:48.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T07:13:48.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:48.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:48.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: Reconfiguring mgr.x (unknown last config time)... 2023-10-09T07:13:48.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T07:13:48.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T07:13:48.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:13:48.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: Reconfiguring daemon mgr.x on smithi115 2023-10-09T07:13:48.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T07:13:48.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:48.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:48 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:13:48.502 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:13:49.636 INFO:teuthology.orchestra.run.smithi115.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T07:13:49.636 INFO:teuthology.orchestra.run.smithi115.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T07:13:49.636 INFO:teuthology.orchestra.run.smithi115.stderr: stderr: 10+0 records in 2023-10-09T07:13:49.636 INFO:teuthology.orchestra.run.smithi115.stderr:10+0 records out 2023-10-09T07:13:49.637 INFO:teuthology.orchestra.run.smithi115.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0111565 s, 940 MB/s 2023-10-09T07:13:49.637 INFO:teuthology.orchestra.run.smithi115.stderr:--> Zapping successful for: 2023-10-09T07:13:50.022 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi115:vg_nvme/lv_4 2023-10-09T07:13:50.238 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:13:50.716 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:50 smithi115 ceph-mon[108061]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T07:13:51.787 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:51 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T07:13:51.787 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:51 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T07:13:51.787 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:51 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:13:52.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:52 smithi115 ceph-mon[108061]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:13:52.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:52 smithi115 ceph-mon[108061]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T07:13:53.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:53 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/256800540' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67d87309-d4ce-4ebf-8ce2-2e24604b56ca"}]: dispatch 2023-10-09T07:13:53.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:53 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/256800540' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "67d87309-d4ce-4ebf-8ce2-2e24604b56ca"}]': finished 2023-10-09T07:13:53.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:53 smithi115 ceph-mon[108061]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T07:13:53.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:53 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T07:13:54.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:54 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/1845722784' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T07:13:54.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:54 smithi115 ceph-mon[108061]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T07:13:56.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:56 smithi115 ceph-mon[108061]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T07:13:58.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:58 smithi115 ceph-mon[108061]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T07:13:58.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:58 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T07:13:58.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:58 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:13:58.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:13:58 smithi115 ceph-mon[108061]: Deploying daemon osd.0 on smithi115 2023-10-09T07:14:00.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:00 smithi115 ceph-mon[108061]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T07:14:02.542 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:02 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:02.543 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:02 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:02.543 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:02 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:14:02.543 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:02 smithi115 ceph-mon[108061]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T07:14:03.493 INFO:teuthology.orchestra.run.smithi115.stdout:Created osd(s) 0 on host 'smithi115' 2023-10-09T07:14:03.874 DEBUG:teuthology.orchestra.run.smithi115:osd.0> sudo journalctl -f -n 0 -u ceph-2817f006-6673-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T07:14:03.876 INFO:tasks.cephadm:Deploying osd.1 on smithi115 with /dev/vg_nvme/lv_3... 2023-10-09T07:14:03.876 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T07:14:03.906 INFO:journalctl@ceph.osd.0.smithi115.stdout:-- Logs begin at Mon 2023-10-09 07:04:09 UTC. -- 2023-10-09T07:14:04.059 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:03 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:04.059 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:03 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:04.060 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:03 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:14:04.060 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:03 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T07:14:04.060 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:03 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:04.060 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:03 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:04.060 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:03 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:04.060 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:03 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:14:04.376 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:04.905 INFO:journalctl@ceph.osd.0.smithi115.stdout:Oct 09 07:14:04 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-0[118606]: 2023-10-09T07:14:04.692+0000 7fdebf352640 -1 osd.0 0 log_to_monitors true 2023-10-09T07:14:04.905 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:04 smithi115 ceph-mon[108061]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T07:14:04.906 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:04 smithi115 ceph-mon[108061]: from='osd.0 [v2:172.21.15.115:6802/2489047323,v1:172.21.15.115:6803/2489047323]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T07:14:05.734 INFO:teuthology.orchestra.run.smithi115.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T07:14:05.734 INFO:teuthology.orchestra.run.smithi115.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T07:14:05.734 INFO:teuthology.orchestra.run.smithi115.stderr: stderr: 10+0 records in 2023-10-09T07:14:05.734 INFO:teuthology.orchestra.run.smithi115.stderr:10+0 records out 2023-10-09T07:14:05.734 INFO:teuthology.orchestra.run.smithi115.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0150476 s, 697 MB/s 2023-10-09T07:14:05.734 INFO:teuthology.orchestra.run.smithi115.stderr:--> Zapping successful for: 2023-10-09T07:14:06.377 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi115:vg_nvme/lv_3 2023-10-09T07:14:06.594 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:06.703 INFO:journalctl@ceph.osd.0.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-0[118606]: 2023-10-09T07:14:06.530+0000 7fdeb49ca700 -1 osd.0 0 waiting for initial osdmap 2023-10-09T07:14:06.704 INFO:journalctl@ceph.osd.0.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-0[118606]: 2023-10-09T07:14:06.534+0000 7fdead7b9700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T07:14:06.704 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-mon[108061]: from='osd.0 [v2:172.21.15.115:6802/2489047323,v1:172.21.15.115:6803/2489047323]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T07:14:06.704 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-mon[108061]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T07:14:06.704 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T07:14:06.704 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-mon[108061]: from='osd.0 [v2:172.21.15.115:6802/2489047323,v1:172.21.15.115:6803/2489047323]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2023-10-09T07:14:06.704 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-mon[108061]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T07:14:06.704 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:06.704 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:06.705 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T07:14:06.705 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:06.705 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:14:06.705 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T07:14:06.705 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:06 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:07.716 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:07 smithi115 ceph-mon[108061]: Detected new or changed devices on smithi115 2023-10-09T07:14:07.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:07 smithi115 ceph-mon[108061]: Adjusting osd_memory_target on smithi115 to 17113M 2023-10-09T07:14:07.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:07 smithi115 ceph-mon[108061]: from='osd.0 [v2:172.21.15.115:6802/2489047323,v1:172.21.15.115:6803/2489047323]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2023-10-09T07:14:07.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:07 smithi115 ceph-mon[108061]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T07:14:07.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:07 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T07:14:07.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:07 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T07:14:08.676 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:08 smithi115 ceph-mon[108061]: purged_snaps scrub starts 2023-10-09T07:14:08.677 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:08 smithi115 ceph-mon[108061]: purged_snaps scrub ok 2023-10-09T07:14:08.677 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:08 smithi115 ceph-mon[108061]: osd.0 [v2:172.21.15.115:6802/2489047323,v1:172.21.15.115:6803/2489047323] boot 2023-10-09T07:14:08.677 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:08 smithi115 ceph-mon[108061]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T07:14:08.677 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:08 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T07:14:08.677 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:08 smithi115 ceph-mon[108061]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T07:14:08.677 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:08 smithi115 ceph-mon[108061]: from='client.14197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:14:08.677 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:08 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T07:14:08.677 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:08 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T07:14:08.679 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:08 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:14:09.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:09 smithi115 ceph-mon[108061]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T07:14:09.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:09 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2730242748' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f4110068-caec-485f-af66-caa47adcdb7d"}]: dispatch 2023-10-09T07:14:09.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:09 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2730242748' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f4110068-caec-485f-af66-caa47adcdb7d"}]': finished 2023-10-09T07:14:09.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:09 smithi115 ceph-mon[108061]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T07:14:09.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:09 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T07:14:10.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:10 smithi115 ceph-mon[108061]: pgmap v22: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-09T07:14:10.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:10 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/3617681712' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T07:14:12.959 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:12 smithi115 ceph-mon[108061]: pgmap v23: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-09T07:14:14.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:14 smithi115 ceph-mon[108061]: pgmap v24: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-09T07:14:14.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:14 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T07:14:14.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:14 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:14:15.881 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:15 smithi115 ceph-mon[108061]: Deploying daemon osd.1 on smithi115 2023-10-09T07:14:16.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:16 smithi115 ceph-mon[108061]: pgmap v25: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-09T07:14:18.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:18 smithi115 ceph-mon[108061]: pgmap v26: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-09T07:14:18.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:18 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:18.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:18 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:18.968 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:18 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:14:19.752 INFO:teuthology.orchestra.run.smithi115.stdout:Created osd(s) 1 on host 'smithi115' 2023-10-09T07:14:20.103 DEBUG:teuthology.orchestra.run.smithi115:osd.1> sudo journalctl -f -n 0 -u ceph-2817f006-6673-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T07:14:20.105 INFO:tasks.cephadm:Waiting for 2 OSDs to come up... 2023-10-09T07:14:20.105 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T07:14:20.136 INFO:journalctl@ceph.osd.1.smithi115.stdout:-- Logs begin at Mon 2023-10-09 07:04:09 UTC. -- 2023-10-09T07:14:20.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:20 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:20.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:20 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:20.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:20 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:14:20.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:20 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T07:14:20.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:20 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:20.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:20 smithi115 ceph-mon[108061]: pgmap v27: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-09T07:14:20.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:20 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:20.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:20 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:20.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:20 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:14:20.650 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:21.175 INFO:journalctl@ceph.osd.1.smithi115.stdout:Oct 09 07:14:20 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-1[122785]: 2023-10-09T07:14:20.916+0000 7f235add3640 -1 osd.1 0 log_to_monitors true 2023-10-09T07:14:21.957 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:21 smithi115 ceph-mon[108061]: from='osd.1 [v2:172.21.15.115:6810/2485804120,v1:172.21.15.115:6811/2485804120]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T07:14:22.023 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:14:22.559 INFO:journalctl@ceph.osd.1.smithi115.stdout:Oct 09 07:14:22 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-1[122785]: 2023-10-09T07:14:22.536+0000 7f235044b700 -1 osd.1 0 waiting for initial osdmap 2023-10-09T07:14:22.560 INFO:journalctl@ceph.osd.1.smithi115.stdout:Oct 09 07:14:22 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-1[122785]: 2023-10-09T07:14:22.539+0000 7f234c240700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T07:14:22.560 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:22 smithi115 ceph-mon[108061]: from='osd.1 [v2:172.21.15.115:6810/2485804120,v1:172.21.15.115:6811/2485804120]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T07:14:22.560 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:22 smithi115 ceph-mon[108061]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T07:14:22.560 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:22 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T07:14:22.560 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:22 smithi115 ceph-mon[108061]: from='osd.1 [v2:172.21.15.115:6810/2485804120,v1:172.21.15.115:6811/2485804120]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2023-10-09T07:14:22.560 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:22 smithi115 ceph-mon[108061]: pgmap v29: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-10-09T07:14:22.560 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:22 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/1829075995' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T07:14:22.561 INFO:teuthology.orchestra.run.smithi115.stdout:{"epoch":11,"num_osds":2,"num_up_osds":1,"osd_up_since":1696835647,"num_in_osds":2,"osd_in_since":1696835649,"num_remapped_pgs":0} 2023-10-09T07:14:23.562 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T07:14:23.793 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:23.908 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: from='osd.1 [v2:172.21.15.115:6810/2485804120,v1:172.21.15.115:6811/2485804120]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2023-10-09T07:14:23.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T07:14:23.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T07:14:23.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T07:14:23.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: Detected new or changed devices on smithi115 2023-10-09T07:14:23.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:23.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:23.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T07:14:23.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T07:14:23.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: Adjusting osd_memory_target on smithi115 to 8556M 2023-10-09T07:14:23.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:23.909 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:14:23.910 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T07:14:23.910 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:23 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:14:24.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:24 smithi115 ceph-mon[108061]: purged_snaps scrub starts 2023-10-09T07:14:24.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:24 smithi115 ceph-mon[108061]: purged_snaps scrub ok 2023-10-09T07:14:24.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:24 smithi115 ceph-mon[108061]: osd.1 [v2:172.21.15.115:6810/2485804120,v1:172.21.15.115:6811/2485804120] boot 2023-10-09T07:14:24.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:24 smithi115 ceph-mon[108061]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T07:14:24.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:24 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T07:14:24.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:24 smithi115 ceph-mon[108061]: pgmap v32: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:24.891 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:14:25.271 INFO:teuthology.orchestra.run.smithi115.stdout:{"epoch":14,"num_osds":2,"num_up_osds":2,"osd_up_since":1696835663,"num_in_osds":2,"osd_in_since":1696835649,"num_remapped_pgs":0} 2023-10-09T07:14:25.272 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T07:14:25.487 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:25.875 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:25 smithi115 ceph-mon[108061]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T07:14:25.875 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:25 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/3716546295' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T07:14:26.587 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:14:26.587 INFO:teuthology.orchestra.run.smithi115.stdout:{"epoch":14,"fsid":"2817f006-6673-11ee-8db5-212e2dc638e7","created":"2023-10-09T07:12:41.167276+0000","modified":"2023-10-09T07:14:24.538396+0000","last_up_change":"2023-10-09T07:14:23.531726+0000","last_in_change":"2023-10-09T07:14:09.360738+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":2,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"67d87309-d4ce-4ebf-8ce2-2e24604b56ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":2489047323},{"type":"v1","addr":"172.21.15.115:6803","nonce":2489047323}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":2489047323},{"type":"v1","addr":"172.21.15.115:6805","nonce":2489047323}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":2489047323},{"type":"v1","addr":"172.21.15.115:6809","nonce":2489047323}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":2489047323},{"type":"v1","addr":"172.21.15.115:6807","nonce":2489047323}]},"public_addr":"172.21.15.115:6803/2489047323","cluster_addr":"172.21.15.115:6805/2489047323","heartbeat_back_addr":"172.21.15.115:6809/2489047323","heartbeat_front_addr":"172.21.15.115:6807/2489047323","state":["exists","up"]},{"osd":1,"uuid":"f4110068-caec-485f-af66-caa47adcdb7d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":2485804120},{"type":"v1","addr":"172.21.15.115:6811","nonce":2485804120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":2485804120},{"type":"v1","addr":"172.21.15.115:6813","nonce":2485804120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":2485804120},{"type":"v1","addr":"172.21.15.115:6817","nonce":2485804120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":2485804120},{"type":"v1","addr":"172.21.15.115:6815","nonce":2485804120}]},"public_addr":"172.21.15.115:6811/2485804120","cluster_addr":"172.21.15.115:6813/2485804120","heartbeat_back_addr":"172.21.15.115:6817/2485804120","heartbeat_front_addr":"172.21.15.115:6815/2485804120","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T07:14:05.699366+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T07:14:21.911189+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.115:0/1333000954":"2023-10-10T07:13:25.645021+0000","172.21.15.115:0/3115933596":"2023-10-10T07:13:25.645021+0000","172.21.15.115:0/3918062057":"2023-10-10T07:13:25.645021+0000","172.21.15.115:0/3699247106":"2023-10-10T07:13:05.264731+0000","172.21.15.115:6800/1108791383":"2023-10-10T07:13:25.645021+0000","172.21.15.115:6800/527469943":"2023-10-10T07:13:05.264731+0000","172.21.15.115:6801/527469943":"2023-10-10T07:13:05.264731+0000","172.21.15.115:6801/1108791383":"2023-10-10T07:13:25.645021+0000","172.21.15.115:0/3318904959":"2023-10-10T07:13:05.264731+0000","172.21.15.115:0/3916126893":"2023-10-10T07:13:05.264731+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T07:14:26.869 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:26 smithi115 ceph-mon[108061]: pgmap v34: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:27.005 INFO:tasks.cephadm.ceph_manager.ceph:[] 2023-10-09T07:14:27.005 INFO:tasks.cephadm:Setting up client nodes... 2023-10-09T07:14:27.006 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T07:14:27.221 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:27.695 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:27 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/337415932' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T07:14:28.383 INFO:teuthology.orchestra.run.smithi115.stdout:[client.0] 2023-10-09T07:14:28.383 INFO:teuthology.orchestra.run.smithi115.stdout: key = AQBUqCNl2xC5FhAAhlvKm/vwujRQP1e8rxYT5Q== 2023-10-09T07:14:28.670 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:28 smithi115 ceph-mon[108061]: pgmap v35: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:28.670 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:28 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/3810451173' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T07:14:28.670 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:28 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/3810451173' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T07:14:28.762 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2023-10-09T07:14:28.763 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-09T07:14:28.763 DEBUG:teuthology.orchestra.run.smithi115:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-09T07:14:28.803 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T07:14:28.803 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T07:14:28.803 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T07:14:29.057 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:30.222 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:14:30.543 INFO:teuthology.orchestra.run.smithi115.stdout:{"epoch":13,"active_gid":14152,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":2424989841},{"type":"v1","addr":"172.21.15.115:6801","nonce":2424989841}]},"active_addr":"172.21.15.115:6801/2424989841","active_change":"2023-10-09T07:13:25.645171+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[],"modules":["cephadm","dashboard","iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.3","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.115:8443/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":3,"active_clients":[{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.115:0","nonce":2823745055}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.115:0","nonce":2849740043}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.115:0","nonce":4287297763}]}]} 2023-10-09T07:14:30.546 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T07:14:30.546 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T07:14:30.546 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T07:14:30.761 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:30.856 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:30 smithi115 ceph-mon[108061]: pgmap v36: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:30.856 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:30 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2715235540' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T07:14:31.814 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:14:31.814 INFO:teuthology.orchestra.run.smithi115.stdout:{"epoch":14,"fsid":"2817f006-6673-11ee-8db5-212e2dc638e7","created":"2023-10-09T07:12:41.167276+0000","modified":"2023-10-09T07:14:24.538396+0000","last_up_change":"2023-10-09T07:14:23.531726+0000","last_in_change":"2023-10-09T07:14:09.360738+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":2,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"67d87309-d4ce-4ebf-8ce2-2e24604b56ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":2489047323},{"type":"v1","addr":"172.21.15.115:6803","nonce":2489047323}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":2489047323},{"type":"v1","addr":"172.21.15.115:6805","nonce":2489047323}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":2489047323},{"type":"v1","addr":"172.21.15.115:6809","nonce":2489047323}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":2489047323},{"type":"v1","addr":"172.21.15.115:6807","nonce":2489047323}]},"public_addr":"172.21.15.115:6803/2489047323","cluster_addr":"172.21.15.115:6805/2489047323","heartbeat_back_addr":"172.21.15.115:6809/2489047323","heartbeat_front_addr":"172.21.15.115:6807/2489047323","state":["exists","up"]},{"osd":1,"uuid":"f4110068-caec-485f-af66-caa47adcdb7d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":2485804120},{"type":"v1","addr":"172.21.15.115:6811","nonce":2485804120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":2485804120},{"type":"v1","addr":"172.21.15.115:6813","nonce":2485804120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":2485804120},{"type":"v1","addr":"172.21.15.115:6817","nonce":2485804120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":2485804120},{"type":"v1","addr":"172.21.15.115:6815","nonce":2485804120}]},"public_addr":"172.21.15.115:6811/2485804120","cluster_addr":"172.21.15.115:6813/2485804120","heartbeat_back_addr":"172.21.15.115:6817/2485804120","heartbeat_front_addr":"172.21.15.115:6815/2485804120","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T07:14:05.699366+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T07:14:21.911189+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.115:0/1333000954":"2023-10-10T07:13:25.645021+0000","172.21.15.115:0/3115933596":"2023-10-10T07:13:25.645021+0000","172.21.15.115:0/3918062057":"2023-10-10T07:13:25.645021+0000","172.21.15.115:0/3699247106":"2023-10-10T07:13:05.264731+0000","172.21.15.115:6800/1108791383":"2023-10-10T07:13:25.645021+0000","172.21.15.115:6800/527469943":"2023-10-10T07:13:05.264731+0000","172.21.15.115:6801/527469943":"2023-10-10T07:13:05.264731+0000","172.21.15.115:6801/1108791383":"2023-10-10T07:13:25.645021+0000","172.21.15.115:0/3318904959":"2023-10-10T07:13:05.264731+0000","172.21.15.115:0/3916126893":"2023-10-10T07:13:05.264731+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T07:14:32.204 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T07:14:32.205 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T07:14:32.419 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:32.869 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:32 smithi115 ceph-mon[108061]: pgmap v37: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:32.869 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:32 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/3957906418' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T07:14:33.494 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:14:33.494 INFO:teuthology.orchestra.run.smithi115.stdout:{"epoch":14,"fsid":"2817f006-6673-11ee-8db5-212e2dc638e7","created":"2023-10-09T07:12:41.167276+0000","modified":"2023-10-09T07:14:24.538396+0000","last_up_change":"2023-10-09T07:14:23.531726+0000","last_in_change":"2023-10-09T07:14:09.360738+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":2,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"67d87309-d4ce-4ebf-8ce2-2e24604b56ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":2489047323},{"type":"v1","addr":"172.21.15.115:6803","nonce":2489047323}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":2489047323},{"type":"v1","addr":"172.21.15.115:6805","nonce":2489047323}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":2489047323},{"type":"v1","addr":"172.21.15.115:6809","nonce":2489047323}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":2489047323},{"type":"v1","addr":"172.21.15.115:6807","nonce":2489047323}]},"public_addr":"172.21.15.115:6803/2489047323","cluster_addr":"172.21.15.115:6805/2489047323","heartbeat_back_addr":"172.21.15.115:6809/2489047323","heartbeat_front_addr":"172.21.15.115:6807/2489047323","state":["exists","up"]},{"osd":1,"uuid":"f4110068-caec-485f-af66-caa47adcdb7d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":2485804120},{"type":"v1","addr":"172.21.15.115:6811","nonce":2485804120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":2485804120},{"type":"v1","addr":"172.21.15.115:6813","nonce":2485804120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":2485804120},{"type":"v1","addr":"172.21.15.115:6817","nonce":2485804120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":2485804120},{"type":"v1","addr":"172.21.15.115:6815","nonce":2485804120}]},"public_addr":"172.21.15.115:6811/2485804120","cluster_addr":"172.21.15.115:6813/2485804120","heartbeat_back_addr":"172.21.15.115:6817/2485804120","heartbeat_front_addr":"172.21.15.115:6815/2485804120","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T07:14:05.699366+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T07:14:21.911189+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.115:0/1333000954":"2023-10-10T07:13:25.645021+0000","172.21.15.115:0/3115933596":"2023-10-10T07:13:25.645021+0000","172.21.15.115:0/3918062057":"2023-10-10T07:13:25.645021+0000","172.21.15.115:0/3699247106":"2023-10-10T07:13:05.264731+0000","172.21.15.115:6800/1108791383":"2023-10-10T07:13:25.645021+0000","172.21.15.115:6800/527469943":"2023-10-10T07:13:05.264731+0000","172.21.15.115:6801/527469943":"2023-10-10T07:13:05.264731+0000","172.21.15.115:6801/1108791383":"2023-10-10T07:13:25.645021+0000","172.21.15.115:0/3318904959":"2023-10-10T07:13:05.264731+0000","172.21.15.115:0/3916126893":"2023-10-10T07:13:05.264731+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T07:14:33.846 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:33 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/1352619307' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T07:14:33.887 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T07:14:33.888 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T07:14:34.112 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:34.162 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:34.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:34 smithi115 ceph-mon[108061]: pgmap v38: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:35.937 INFO:teuthology.orchestra.run.smithi115.stdout:34359738375 2023-10-09T07:14:35.937 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T07:14:36.209 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:36.253 INFO:teuthology.orchestra.run.smithi115.stdout:55834574853 2023-10-09T07:14:36.253 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T07:14:36.572 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:36.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:36 smithi115 ceph-mon[108061]: pgmap v39: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:37.944 INFO:teuthology.orchestra.run.smithi115.stdout:34359738375 2023-10-09T07:14:38.166 INFO:teuthology.orchestra.run.smithi115.stdout:55834574853 2023-10-09T07:14:38.379 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738375 got 34359738375 for osd.0 2023-10-09T07:14:38.379 DEBUG:teuthology.parallel:result is None 2023-10-09T07:14:38.610 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574853 got 55834574853 for osd.1 2023-10-09T07:14:38.610 DEBUG:teuthology.parallel:result is None 2023-10-09T07:14:38.611 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T07:14:38.611 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T07:14:38.828 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:38.946 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:38 smithi115 ceph-mon[108061]: pgmap v40: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:38.946 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:38 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2717515500' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T07:14:38.946 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:38 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/4151011370' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T07:14:39.911 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:14:39.913 INFO:teuthology.orchestra.run.smithi115.stderr:dumped all 2023-10-09T07:14:40.293 INFO:teuthology.orchestra.run.smithi115.stdout:{"pg_ready":true,"pg_map":{"version":41,"stamp":"2023-10-09T07:14:39.659575+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":0,"num_osds":2,"num_per_pool_osds":2,"num_per_pool_omap_osds":0,"kb":187490304,"kb_used":54280,"kb_used_data":240,"kb_used_omap":3,"kb_used_meta":54012,"kb_avail":187436024,"statfs":{"total":191990071296,"available":191934488576,"internally_reserved":0,"allocated":245760,"data_stored":59464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3562,"internal_metadata":55308822},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[],"pool_stats":[],"osd_stats":[{"osd":1,"up_from":13,"seq":55834574854,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27140,"kb_used_data":120,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93718012,"statfs":{"total":95995035648,"available":95967244288,"internally_reserved":0,"allocated":122880,"data_stored":29732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46400000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738376,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27140,"kb_used_data":120,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93718012,"statfs":{"total":95995035648,"available":95967244288,"internally_reserved":0,"allocated":122880,"data_stored":29732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25900000000000001}]}]}],"pool_statfs":[]}} 2023-10-09T07:14:40.294 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T07:14:40.508 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:40.906 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:40 smithi115 ceph-mon[108061]: pgmap v41: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:40.906 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:40 smithi115 ceph-mon[108061]: from='client.14228 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T07:14:41.597 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:14:41.599 INFO:teuthology.orchestra.run.smithi115.stderr:dumped all 2023-10-09T07:14:41.977 INFO:teuthology.orchestra.run.smithi115.stdout:{"pg_ready":true,"pg_map":{"version":41,"stamp":"2023-10-09T07:14:39.659575+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":0,"num_osds":2,"num_per_pool_osds":2,"num_per_pool_omap_osds":0,"kb":187490304,"kb_used":54280,"kb_used_data":240,"kb_used_omap":3,"kb_used_meta":54012,"kb_avail":187436024,"statfs":{"total":191990071296,"available":191934488576,"internally_reserved":0,"allocated":245760,"data_stored":59464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3562,"internal_metadata":55308822},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[],"pool_stats":[],"osd_stats":[{"osd":1,"up_from":13,"seq":55834574854,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27140,"kb_used_data":120,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93718012,"statfs":{"total":95995035648,"available":95967244288,"internally_reserved":0,"allocated":122880,"data_stored":29732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46400000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738376,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":27140,"kb_used_data":120,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93718012,"statfs":{"total":95995035648,"available":95967244288,"internally_reserved":0,"allocated":122880,"data_stored":29732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25900000000000001}]}]}],"pool_statfs":[]}} 2023-10-09T07:14:41.977 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T07:14:41.977 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T07:14:41.978 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T07:14:41.978 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T07:14:42.194 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:42.958 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:42 smithi115 ceph-mon[108061]: from='client.14230 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T07:14:42.958 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:42 smithi115 ceph-mon[108061]: pgmap v42: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:43.312 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:14:43.313 INFO:teuthology.orchestra.run.smithi115.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T07:14:43.719 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T07:14:43.719 INFO:tasks.cephadm:Setup complete, yielding 2023-10-09T07:14:43.720 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-09T07:14:43.731 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi115.front.sepia.ceph.com 2023-10-09T07:14:43.732 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch status' 2023-10-09T07:14:43.871 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:43 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2272578217' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T07:14:43.947 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:45.054 INFO:teuthology.orchestra.run.smithi115.stdout:Backend: cephadm 2023-10-09T07:14:45.054 INFO:teuthology.orchestra.run.smithi115.stdout:Available: Yes 2023-10-09T07:14:45.054 INFO:teuthology.orchestra.run.smithi115.stdout:Paused: No 2023-10-09T07:14:45.078 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:44 smithi115 ceph-mon[108061]: pgmap v43: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:45.442 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch ps' 2023-10-09T07:14:45.656 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:46.727 INFO:teuthology.orchestra.run.smithi115.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T07:14:46.727 INFO:teuthology.orchestra.run.smithi115.stdout:mgr.x smithi115 *:9283,8765 running (2m) - 119s - - 18.0.0-5817-gbcea8c48 fa922c444906 0af3be2fd030 2023-10-09T07:14:46.727 INFO:teuthology.orchestra.run.smithi115.stdout:mon.a smithi115 running (2m) - 2m - 2048M 18.0.0-5817-gbcea8c48 fa922c444906 bb6211c91791 2023-10-09T07:14:46.727 INFO:teuthology.orchestra.run.smithi115.stdout:osd.0 smithi115 running (45s) - 45s - 8556M 18.0.0-5817-gbcea8c48 fa922c444906 e9c47d0a77e2 2023-10-09T07:14:46.727 INFO:teuthology.orchestra.run.smithi115.stdout:osd.1 smithi115 running (29s) - 29s - 8556M 18.0.0-5817-gbcea8c48 fa922c444906 d54e8efe09c7 2023-10-09T07:14:47.033 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:46 smithi115 ceph-mon[108061]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:14:47.033 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:46 smithi115 ceph-mon[108061]: pgmap v44: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:47.055 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch ls' 2023-10-09T07:14:47.268 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:48.029 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:47 smithi115 ceph-mon[108061]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:14:48.400 INFO:teuthology.orchestra.run.smithi115.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2023-10-09T07:14:48.400 INFO:teuthology.orchestra.run.smithi115.stdout:mgr 1/1 - 61s smithi115=x;count:1 2023-10-09T07:14:48.400 INFO:teuthology.orchestra.run.smithi115.stdout:mon 1/1 - 67s smithi115:172.21.15.115=a;count:1 2023-10-09T07:14:48.400 INFO:teuthology.orchestra.run.smithi115.stdout:osd 2 - - 2023-10-09T07:14:48.783 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch host ls' 2023-10-09T07:14:48.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:48 smithi115 ceph-mon[108061]: pgmap v45: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:48.997 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:49.873 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:49 smithi115 ceph-mon[108061]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:14:50.095 INFO:teuthology.orchestra.run.smithi115.stdout:HOST ADDR LABELS STATUS 2023-10-09T07:14:50.095 INFO:teuthology.orchestra.run.smithi115.stdout:smithi115 172.21.15.115 2023-10-09T07:14:50.095 INFO:teuthology.orchestra.run.smithi115.stdout:1 hosts in cluster 2023-10-09T07:14:50.458 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch device ls' 2023-10-09T07:14:50.673 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:51.207 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:50 smithi115 ceph-mon[108061]: pgmap v46: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:51.824 INFO:teuthology.orchestra.run.smithi115.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REFRESHED REJECT REASONS 2023-10-09T07:14:51.824 INFO:teuthology.orchestra.run.smithi115.stdout:smithi115 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400UU400BGN 372G No 29s ago LVM detected, locked 2023-10-09T07:14:52.028 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:51 smithi115 ceph-mon[108061]: from='client.14240 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:14:52.225 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph osd lspools' 2023-10-09T07:14:52.438 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:52.902 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:52 smithi115 ceph-mon[108061]: pgmap v47: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:52.903 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:52 smithi115 ceph-mon[108061]: from='client.14242 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:14:53.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:53 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/1338451475' entity='client.admin' cmd=[{"prefix": "osd lspools"}]: dispatch 2023-10-09T07:14:54.021 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph osd pool create mypool' 2023-10-09T07:14:54.235 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:55.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:54 smithi115 ceph-mon[108061]: pgmap v48: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:55.756 INFO:teuthology.orchestra.run.smithi115.stderr:pool 'mypool' created 2023-10-09T07:14:56.072 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:55 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2630846565' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "mypool"}]: dispatch 2023-10-09T07:14:56.139 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph osd pool application enable mypool rbd' 2023-10-09T07:14:56.354 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:57.053 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:56 smithi115 ceph-mon[108061]: pgmap v49: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:57.054 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:56 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2630846565' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "mypool"}]': finished 2023-10-09T07:14:57.054 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:56 smithi115 ceph-mon[108061]: osdmap e15: 2 total, 2 up, 2 in 2023-10-09T07:14:57.754 INFO:teuthology.orchestra.run.smithi115.stderr:enabled application 'rbd' on pool 'mypool' 2023-10-09T07:14:58.097 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:57 smithi115 ceph-mon[108061]: osdmap e16: 2 total, 2 up, 2 in 2023-10-09T07:14:58.097 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:57 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/4110242839' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "mypool", "app": "rbd"}]: dispatch 2023-10-09T07:14:58.163 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph osd lspools' 2023-10-09T07:14:58.379 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:14:59.094 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:58 smithi115 ceph-mon[108061]: pgmap v52: 32 pgs: 32 unknown; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:14:59.094 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:58 smithi115 ceph-mon[108061]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T07:14:59.094 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:58 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/4110242839' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "mypool", "app": "rbd"}]': finished 2023-10-09T07:14:59.094 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:58 smithi115 ceph-mon[108061]: osdmap e17: 2 total, 2 up, 2 in 2023-10-09T07:14:59.469 INFO:teuthology.orchestra.run.smithi115.stdout:1 mypool 2023-10-09T07:14:59.856 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph osd pool ls detail' 2023-10-09T07:15:00.074 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:15:00.182 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:14:59 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/466216832' entity='client.admin' cmd=[{"prefix": "osd lspools"}]: dispatch 2023-10-09T07:15:00.874 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:00 smithi115 ceph-mon[108061]: pgmap v54: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:00.874 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:00 smithi115 ceph-mon[108061]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T07:15:00.874 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:00 smithi115 ceph-mon[108061]: Cluster is now healthy 2023-10-09T07:15:01.153 INFO:teuthology.orchestra.run.smithi115.stdout:pool 1 'mypool' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 17 flags hashpspool stripe_width 0 application rbd read_balance_score 1.44 2023-10-09T07:15:01.153 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:15:01.547 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'sleep 60' 2023-10-09T07:15:01.763 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:15:02.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:01 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/1052908249' entity='client.admin' cmd=[{"prefix": "osd pool ls", "detail": "detail"}]: dispatch 2023-10-09T07:15:03.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:02 smithi115 ceph-mon[108061]: pgmap v55: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:05.218 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:04 smithi115 ceph-mon[108061]: pgmap v56: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:07.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:06 smithi115 ceph-mon[108061]: pgmap v57: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:09.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:08 smithi115 ceph-mon[108061]: pgmap v58: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:10.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:10 smithi115 ceph-mon[108061]: pgmap v59: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:12.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:12 smithi115 ceph-mon[108061]: pgmap v60: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:15.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:14 smithi115 ceph-mon[108061]: pgmap v61: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:17.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:16 smithi115 ceph-mon[108061]: pgmap v62: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:19.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:18 smithi115 ceph-mon[108061]: pgmap v63: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:20.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:20 smithi115 ceph-mon[108061]: pgmap v64: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:23.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:22 smithi115 ceph-mon[108061]: pgmap v65: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:23.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:22 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:15:24.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:23 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:15:24.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:23 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T07:15:24.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:23 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:15:25.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:24 smithi115 ceph-mon[108061]: pgmap v66: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:25.966 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:15:25 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:15:25.672+0000 7f2cb88ef700 -1 calc_pg_upmaps abort due to max <= 0 2023-10-09T07:15:27.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:26 smithi115 ceph-mon[108061]: pgmap v67: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:29.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:28 smithi115 ceph-mon[108061]: pgmap v68: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:30.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:30 smithi115 ceph-mon[108061]: pgmap v69: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:33.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:32 smithi115 ceph-mon[108061]: pgmap v70: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:35.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:34 smithi115 ceph-mon[108061]: pgmap v71: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:37.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:36 smithi115 ceph-mon[108061]: pgmap v72: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:39.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:38 smithi115 ceph-mon[108061]: pgmap v73: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:40.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:40 smithi115 ceph-mon[108061]: pgmap v74: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:43.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:42 smithi115 ceph-mon[108061]: pgmap v75: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:45.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:44 smithi115 ceph-mon[108061]: pgmap v76: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:47.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:46 smithi115 ceph-mon[108061]: pgmap v77: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:49.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:48 smithi115 ceph-mon[108061]: pgmap v78: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:50.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:50 smithi115 ceph-mon[108061]: pgmap v79: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:53.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:52 smithi115 ceph-mon[108061]: pgmap v80: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:55.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:54 smithi115 ceph-mon[108061]: pgmap v81: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:57.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:56 smithi115 ceph-mon[108061]: pgmap v82: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:15:58.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:57 smithi115 ceph-mon[108061]: Health check failed: Degraded data redundancy: 32 pgs undersized (PG_DEGRADED) 2023-10-09T07:15:59.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:15:58 smithi115 ceph-mon[108061]: pgmap v83: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:00.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:00 smithi115 ceph-mon[108061]: pgmap v84: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:02.886 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:02 smithi115 ceph-mon[108061]: pgmap v85: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:02.888 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mgr mgr/cephadm/container_image_nvmeof quay.io/ceph/nvmeof:0.0.4' 2023-10-09T07:16:03.111 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:16:04.666 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'sleep 60' 2023-10-09T07:16:04.883 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:16:04.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:04 smithi115 ceph-mon[108061]: pgmap v86: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:04.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:04 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/1127712757' entity='client.admin' 2023-10-09T07:16:04.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:04 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:16:04.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:04 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:16:04.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:04 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T07:16:04.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:04 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:16:07.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:06 smithi115 ceph-mon[108061]: pgmap v87: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:09.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:08 smithi115 ceph-mon[108061]: pgmap v88: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:10.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:10 smithi115 ceph-mon[108061]: pgmap v89: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:13.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:12 smithi115 ceph-mon[108061]: pgmap v90: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:15.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:14 smithi115 ceph-mon[108061]: pgmap v91: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:17.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:16 smithi115 ceph-mon[108061]: pgmap v92: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:19.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:18 smithi115 ceph-mon[108061]: pgmap v93: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:20.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:20 smithi115 ceph-mon[108061]: pgmap v94: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:23.218 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:22 smithi115 ceph-mon[108061]: pgmap v95: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:25.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:24 smithi115 ceph-mon[108061]: pgmap v96: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:25.967 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:16:25 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:16:25.674+0000 7f2cb88ef700 -1 calc_pg_upmaps abort due to max <= 0 2023-10-09T07:16:27.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:26 smithi115 ceph-mon[108061]: pgmap v97: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:29.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:28 smithi115 ceph-mon[108061]: pgmap v98: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:30.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:30 smithi115 ceph-mon[108061]: pgmap v99: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:33.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:32 smithi115 ceph-mon[108061]: pgmap v100: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:35.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:34 smithi115 ceph-mon[108061]: pgmap v101: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:37.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:36 smithi115 ceph-mon[108061]: pgmap v102: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:39.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:38 smithi115 ceph-mon[108061]: pgmap v103: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:40.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:40 smithi115 ceph-mon[108061]: pgmap v104: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:43.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:42 smithi115 ceph-mon[108061]: pgmap v105: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:45.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:44 smithi115 ceph-mon[108061]: pgmap v106: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:47.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:46 smithi115 ceph-mon[108061]: pgmap v107: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:49.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:48 smithi115 ceph-mon[108061]: pgmap v108: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:50.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:50 smithi115 ceph-mon[108061]: pgmap v109: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:53.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:52 smithi115 ceph-mon[108061]: pgmap v110: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:55.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:54 smithi115 ceph-mon[108061]: pgmap v111: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:57.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:56 smithi115 ceph-mon[108061]: pgmap v112: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:16:59.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:16:58 smithi115 ceph-mon[108061]: pgmap v113: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:00.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:00 smithi115 ceph-mon[108061]: pgmap v114: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:03.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:02 smithi115 ceph-mon[108061]: pgmap v115: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:05.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:04 smithi115 ceph-mon[108061]: pgmap v116: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:05.217 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:04 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:17:06.019 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:05 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:17:06.019 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:05 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T07:17:06.019 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:05 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:06.074 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config get mgr mgr/cephadm/container_image_nvmeof' 2023-10-09T07:17:06.295 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:17:07.073 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:06 smithi115 ceph-mon[108061]: pgmap v117: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:07.451 INFO:teuthology.orchestra.run.smithi115.stdout:quay.io/ceph/nvmeof:0.0.4 2023-10-09T07:17:07.835 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch apply nvmeof mypool --placement="1 $(hostname)"' 2023-10-09T07:17:08.051 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:17:08.179 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:07 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2971634841' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/container_image_nvmeof"}]: dispatch 2023-10-09T07:17:08.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:08 smithi115 ceph-mon[108061]: pgmap v118: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:09.187 INFO:teuthology.orchestra.run.smithi115.stdout:Scheduled nvmeof.mypool update... 2023-10-09T07:17:09.572 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch ps --refresh' 2023-10-09T07:17:09.802 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:17:10.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:10 smithi115 ceph-mon[108061]: from='client.14258 -' entity='client.admin' cmd=[{"prefix": "orch apply nvmeof", "pool": "mypool", "placement": "1 smithi115", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:17:10.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:10 smithi115 ceph-mon[108061]: Saving service nvmeof.mypool spec with placement smithi115;count:1 2023-10-09T07:17:10.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:10 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:10.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:10 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:17:10.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:10 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:17:10.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:10 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T07:17:10.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:10 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:10.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:10 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.nvmeof.mypool.smithi115.rymvww", "caps": ["mon", "profile rbd", "osd", "allow all tag rbd *=*"]}]: dispatch 2023-10-09T07:17:10.469 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:10 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "client.nvmeof.mypool.smithi115.rymvww", "caps": ["mon", "profile rbd", "osd", "allow all tag rbd *=*"]}]': finished 2023-10-09T07:17:10.469 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:10 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:17:10.469 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:10 smithi115 ceph-mon[108061]: Deploying daemon nvmeof.mypool.smithi115.rymvww on smithi115 2023-10-09T07:17:10.469 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:10 smithi115 ceph-mon[108061]: pgmap v119: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:10.958 INFO:teuthology.orchestra.run.smithi115.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T07:17:10.958 INFO:teuthology.orchestra.run.smithi115.stdout:mgr.x smithi115 *:9283,8765 running (4m) - 4m - - 18.0.0-5817-gbcea8c48 fa922c444906 0af3be2fd030 2023-10-09T07:17:10.958 INFO:teuthology.orchestra.run.smithi115.stdout:mon.a smithi115 running (4m) - 4m - 2048M 18.0.0-5817-gbcea8c48 fa922c444906 bb6211c91791 2023-10-09T07:17:10.958 INFO:teuthology.orchestra.run.smithi115.stdout:osd.0 smithi115 running (3m) - 3m - 8556M 18.0.0-5817-gbcea8c48 fa922c444906 e9c47d0a77e2 2023-10-09T07:17:10.958 INFO:teuthology.orchestra.run.smithi115.stdout:osd.1 smithi115 running (2m) - 2m - 8556M 18.0.0-5817-gbcea8c48 fa922c444906 d54e8efe09c7 2023-10-09T07:17:11.490 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph versions' 2023-10-09T07:17:11.764 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:17:11.913 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:11 smithi115 ceph-mon[108061]: from='client.14260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:17:12.630 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:12 smithi115 ceph-mon[108061]: pgmap v120: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:13.188 INFO:teuthology.orchestra.run.smithi115.stdout:{ 2023-10-09T07:17:13.188 INFO:teuthology.orchestra.run.smithi115.stdout: "mon": { 2023-10-09T07:17:13.189 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev)": 1 2023-10-09T07:17:13.189 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2023-10-09T07:17:13.189 INFO:teuthology.orchestra.run.smithi115.stdout: "mgr": { 2023-10-09T07:17:13.189 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev)": 1 2023-10-09T07:17:13.189 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2023-10-09T07:17:13.189 INFO:teuthology.orchestra.run.smithi115.stdout: "osd": { 2023-10-09T07:17:13.189 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev)": 2 2023-10-09T07:17:13.189 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2023-10-09T07:17:13.189 INFO:teuthology.orchestra.run.smithi115.stdout: "overall": { 2023-10-09T07:17:13.189 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph version 18.0.0-5817-gbcea8c48 (bcea8c48517ff154f392584022efa0718f51c9f2) reef (dev)": 4 2023-10-09T07:17:13.189 INFO:teuthology.orchestra.run.smithi115.stdout: } 2023-10-09T07:17:13.189 INFO:teuthology.orchestra.run.smithi115.stdout:} 2023-10-09T07:17:13.579 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph -s' 2023-10-09T07:17:13.967 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:13 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2512072502' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T07:17:14.966 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:14 smithi115 ceph-mon[108061]: pgmap v121: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:17.216 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:16 smithi115 ceph-mon[108061]: pgmap v122: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:17.902 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:17:19.083 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:18 smithi115 ceph-mon[108061]: pgmap v123: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:19.802 INFO:teuthology.orchestra.run.smithi115.stdout: cluster: 2023-10-09T07:17:19.802 INFO:teuthology.orchestra.run.smithi115.stdout: id: 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:17:19.802 INFO:teuthology.orchestra.run.smithi115.stdout: health: HEALTH_WARN 2023-10-09T07:17:19.802 INFO:teuthology.orchestra.run.smithi115.stdout: Degraded data redundancy: 32 pgs undersized 2023-10-09T07:17:19.802 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:17:19.802 INFO:teuthology.orchestra.run.smithi115.stdout: services: 2023-10-09T07:17:19.803 INFO:teuthology.orchestra.run.smithi115.stdout: mon: 1 daemons, quorum a (age 4m) 2023-10-09T07:17:19.803 INFO:teuthology.orchestra.run.smithi115.stdout: mgr: x(active, since 3m) 2023-10-09T07:17:19.803 INFO:teuthology.orchestra.run.smithi115.stdout: osd: 2 osds: 2 up (since 2m), 2 in (since 3m) 2023-10-09T07:17:19.803 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:17:19.803 INFO:teuthology.orchestra.run.smithi115.stdout: data: 2023-10-09T07:17:19.803 INFO:teuthology.orchestra.run.smithi115.stdout: pools: 1 pools, 32 pgs 2023-10-09T07:17:19.803 INFO:teuthology.orchestra.run.smithi115.stdout: objects: 0 objects, 0 B 2023-10-09T07:17:19.803 INFO:teuthology.orchestra.run.smithi115.stdout: usage: 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:19.803 INFO:teuthology.orchestra.run.smithi115.stdout: pgs: 32 active+undersized 2023-10-09T07:17:19.803 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:17:19.804 INFO:teuthology.orchestra.run.smithi115.stdout: progress: 2023-10-09T07:17:19.804 INFO:teuthology.orchestra.run.smithi115.stdout: Global Recovery Event (0s) 2023-10-09T07:17:19.804 INFO:teuthology.orchestra.run.smithi115.stdout: [............................] 2023-10-09T07:17:19.804 INFO:teuthology.orchestra.run.smithi115.stdout: Updating nvmeof.mypool deployment (+1 -> 1) (0s) 2023-10-09T07:17:19.804 INFO:teuthology.orchestra.run.smithi115.stdout: [............................] 2023-10-09T07:17:19.804 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:17:20.210 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'ceph orch ls' 2023-10-09T07:17:20.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:20 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:20.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:20 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:20.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:20 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:20.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:20 smithi115 ceph-mon[108061]: Checking pool "mypool" exists for service nvmeof.mypool 2023-10-09T07:17:20.467 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:20 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:20.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:20 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:17:20.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:20 smithi115 ceph-mon[108061]: pgmap v124: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:20.468 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:20 smithi115 ceph-mon[108061]: from='client.? 172.21.15.115:0/2445717542' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T07:17:20.722 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:17:21.716 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:21 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:21.716 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:21 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:21.716 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:21 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:21.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:21 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:17:21.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:21 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T07:17:21.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:21 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:21.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:21 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T07:17:21.869 INFO:teuthology.orchestra.run.smithi115.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2023-10-09T07:17:21.870 INFO:teuthology.orchestra.run.smithi115.stdout:mgr 1/1 - 3m smithi115=x;count:1 2023-10-09T07:17:21.870 INFO:teuthology.orchestra.run.smithi115.stdout:mon 1/1 - 3m smithi115:172.21.15.115=a;count:1 2023-10-09T07:17:21.870 INFO:teuthology.orchestra.run.smithi115.stdout:nvmeof.mypool ?:4420,5500,8009 0/1 - 12s smithi115;count:1 2023-10-09T07:17:21.870 INFO:teuthology.orchestra.run.smithi115.stdout:osd 2 - - 2023-10-09T07:17:22.236 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'rbd create mypool/myimage --size 8Gi' 2023-10-09T07:17:22.514 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:17:22.776 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:22 smithi115 ceph-mon[108061]: pgmap v125: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:22.776 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:22 smithi115 ceph-mon[108061]: mgrmap e14: x(active, since 3m) 2023-10-09T07:17:22.776 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:22 smithi115 ceph-mon[108061]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T07:17:22.776 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:22 smithi115 ceph-mon[108061]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-10-09T07:17:24.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:24 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:24.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:24 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:24.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:24 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T07:17:24.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:24 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T07:17:24.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:24 smithi115 ceph-mon[108061]: pgmap v126: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:24.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:24 smithi115 ceph-mon[108061]: pgmap v127: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:24.717 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:24 smithi115 ceph-mon[108061]: from='mgr.14152 172.21.15.115:0/347380656' entity='mgr.x' 2023-10-09T07:17:25.678 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:25 smithi115 ceph-mon[108061]: osdmap e18: 2 total, 2 up, 2 in 2023-10-09T07:17:25.782 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'rbd ls mypool' 2023-10-09T07:17:25.966 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:17:25 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x[108285]: 2023-10-09T07:17:25.676+0000 7f2cb88ef700 -1 calc_pg_upmaps abort due to max <= 0 2023-10-09T07:17:26.007 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:17:26.710 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:26 smithi115 ceph-mon[108061]: pgmap v129: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:26.711 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:26 smithi115 ceph-mon[108061]: osdmap e19: 2 total, 2 up, 2 in 2023-10-09T07:17:26.817 INFO:teuthology.orchestra.run.smithi115.stdout:myimage 2023-10-09T07:17:27.190 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c 'set -ex 2023-10-09T07:17:27.190 DEBUG:teuthology.orchestra.run.smithi115:> # 2023-10-09T07:17:27.191 DEBUG:teuthology.orchestra.run.smithi115:> cat /proc/cpuinfo 2023-10-09T07:17:27.191 DEBUG:teuthology.orchestra.run.smithi115:> # print nvmeof start up logs 2023-10-09T07:17:27.191 DEBUG:teuthology.orchestra.run.smithi115:> s=$(systemctl list-units | grep nvmeof.mypool | awk '"'"'{print $1}'"'"') 2023-10-09T07:17:27.191 DEBUG:teuthology.orchestra.run.smithi115:> echo "<---- s variable is [ $s ]---->" 2023-10-09T07:17:27.191 DEBUG:teuthology.orchestra.run.smithi115:> PAGER=cat journalctl -u $s 2023-10-09T07:17:27.191 DEBUG:teuthology.orchestra.run.smithi115:> ' 2023-10-09T07:17:27.406 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T07:17:28.275 INFO:teuthology.orchestra.run.smithi115.stderr:+ cat /proc/cpuinfo 2023-10-09T07:17:28.275 INFO:teuthology.orchestra.run.smithi115.stderr:++ systemctl list-units 2023-10-09T07:17:28.276 INFO:teuthology.orchestra.run.smithi115.stdout:processor : 0 2023-10-09T07:17:28.276 INFO:teuthology.orchestra.run.smithi115.stdout:vendor_id : GenuineIntel 2023-10-09T07:17:28.276 INFO:teuthology.orchestra.run.smithi115.stdout:cpu family : 6 2023-10-09T07:17:28.276 INFO:teuthology.orchestra.run.smithi115.stdout:model : 79 2023-10-09T07:17:28.276 INFO:teuthology.orchestra.run.smithi115.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz 2023-10-09T07:17:28.276 INFO:teuthology.orchestra.run.smithi115.stdout:stepping : 1 2023-10-09T07:17:28.276 INFO:teuthology.orchestra.run.smithi115.stdout:microcode : 0xb000010 2023-10-09T07:17:28.276 INFO:teuthology.orchestra.run.smithi115.stdout:cpu MHz : 3800.000 2023-10-09T07:17:28.276 INFO:teuthology.orchestra.run.smithi115.stdout:cache size : 10240 KB 2023-10-09T07:17:28.276 INFO:teuthology.orchestra.run.smithi115.stdout:physical id : 0 2023-10-09T07:17:28.277 INFO:teuthology.orchestra.run.smithi115.stdout:siblings : 8 2023-10-09T07:17:28.277 INFO:teuthology.orchestra.run.smithi115.stdout:core id : 0 2023-10-09T07:17:28.277 INFO:teuthology.orchestra.run.smithi115.stdout:cpu cores : 4 2023-10-09T07:17:28.277 INFO:teuthology.orchestra.run.smithi115.stdout:apicid : 0 2023-10-09T07:17:28.277 INFO:teuthology.orchestra.run.smithi115.stdout:initial apicid : 0 2023-10-09T07:17:28.277 INFO:teuthology.orchestra.run.smithi115.stdout:fpu : yes 2023-10-09T07:17:28.277 INFO:teuthology.orchestra.run.smithi115.stdout:fpu_exception : yes 2023-10-09T07:17:28.277 INFO:teuthology.orchestra.run.smithi115.stdout:cpuid level : 20 2023-10-09T07:17:28.277 INFO:teuthology.orchestra.run.smithi115.stdout:wp : yes 2023-10-09T07:17:28.277 INFO:teuthology.orchestra.run.smithi115.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single pti intel_ppin tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdt_a rdseed adx smap intel_pt xsaveopt cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts 2023-10-09T07:17:28.277 INFO:teuthology.orchestra.run.smithi115.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa itlb_multihit mmio_stale_data 2023-10-09T07:17:28.277 INFO:teuthology.orchestra.run.smithi115.stdout:bogomips : 7000.35 2023-10-09T07:17:28.278 INFO:teuthology.orchestra.run.smithi115.stdout:clflush size : 64 2023-10-09T07:17:28.278 INFO:teuthology.orchestra.run.smithi115.stdout:cache_alignment : 64 2023-10-09T07:17:28.278 INFO:teuthology.orchestra.run.smithi115.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-09T07:17:28.278 INFO:teuthology.orchestra.run.smithi115.stdout:power management: 2023-10-09T07:17:28.278 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:17:28.278 INFO:teuthology.orchestra.run.smithi115.stdout:processor : 1 2023-10-09T07:17:28.278 INFO:teuthology.orchestra.run.smithi115.stdout:vendor_id : GenuineIntel 2023-10-09T07:17:28.278 INFO:teuthology.orchestra.run.smithi115.stdout:cpu family : 6 2023-10-09T07:17:28.278 INFO:teuthology.orchestra.run.smithi115.stdout:model : 79 2023-10-09T07:17:28.278 INFO:teuthology.orchestra.run.smithi115.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz 2023-10-09T07:17:28.278 INFO:teuthology.orchestra.run.smithi115.stdout:stepping : 1 2023-10-09T07:17:28.279 INFO:teuthology.orchestra.run.smithi115.stdout:microcode : 0xb000010 2023-10-09T07:17:28.279 INFO:teuthology.orchestra.run.smithi115.stdout:cpu MHz : 3800.000 2023-10-09T07:17:28.279 INFO:teuthology.orchestra.run.smithi115.stdout:cache size : 10240 KB 2023-10-09T07:17:28.279 INFO:teuthology.orchestra.run.smithi115.stdout:physical id : 0 2023-10-09T07:17:28.279 INFO:teuthology.orchestra.run.smithi115.stdout:siblings : 8 2023-10-09T07:17:28.279 INFO:teuthology.orchestra.run.smithi115.stdout:core id : 1 2023-10-09T07:17:28.279 INFO:teuthology.orchestra.run.smithi115.stdout:cpu cores : 4 2023-10-09T07:17:28.279 INFO:teuthology.orchestra.run.smithi115.stdout:apicid : 2 2023-10-09T07:17:28.279 INFO:teuthology.orchestra.run.smithi115.stdout:initial apicid : 2 2023-10-09T07:17:28.279 INFO:teuthology.orchestra.run.smithi115.stdout:fpu : yes 2023-10-09T07:17:28.279 INFO:teuthology.orchestra.run.smithi115.stdout:fpu_exception : yes 2023-10-09T07:17:28.280 INFO:teuthology.orchestra.run.smithi115.stdout:cpuid level : 20 2023-10-09T07:17:28.280 INFO:teuthology.orchestra.run.smithi115.stdout:wp : yes 2023-10-09T07:17:28.280 INFO:teuthology.orchestra.run.smithi115.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single pti intel_ppin tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdt_a rdseed adx smap intel_pt xsaveopt cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts 2023-10-09T07:17:28.280 INFO:teuthology.orchestra.run.smithi115.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa itlb_multihit mmio_stale_data 2023-10-09T07:17:28.280 INFO:teuthology.orchestra.run.smithi115.stdout:bogomips : 7000.35 2023-10-09T07:17:28.280 INFO:teuthology.orchestra.run.smithi115.stdout:clflush size : 64 2023-10-09T07:17:28.280 INFO:teuthology.orchestra.run.smithi115.stdout:cache_alignment : 64 2023-10-09T07:17:28.280 INFO:teuthology.orchestra.run.smithi115.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-09T07:17:28.280 INFO:teuthology.orchestra.run.smithi115.stdout:power management: 2023-10-09T07:17:28.280 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:17:28.280 INFO:teuthology.orchestra.run.smithi115.stdout:processor : 2 2023-10-09T07:17:28.281 INFO:teuthology.orchestra.run.smithi115.stdout:vendor_id : GenuineIntel 2023-10-09T07:17:28.281 INFO:teuthology.orchestra.run.smithi115.stdout:cpu family : 6 2023-10-09T07:17:28.281 INFO:teuthology.orchestra.run.smithi115.stdout:model : 79 2023-10-09T07:17:28.281 INFO:teuthology.orchestra.run.smithi115.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz 2023-10-09T07:17:28.281 INFO:teuthology.orchestra.run.smithi115.stdout:stepping : 1 2023-10-09T07:17:28.281 INFO:teuthology.orchestra.run.smithi115.stdout:microcode : 0xb000010 2023-10-09T07:17:28.281 INFO:teuthology.orchestra.run.smithi115.stdout:cpu MHz : 3800.000 2023-10-09T07:17:28.281 INFO:teuthology.orchestra.run.smithi115.stdout:cache size : 10240 KB 2023-10-09T07:17:28.281 INFO:teuthology.orchestra.run.smithi115.stdout:physical id : 0 2023-10-09T07:17:28.281 INFO:teuthology.orchestra.run.smithi115.stdout:siblings : 8 2023-10-09T07:17:28.281 INFO:teuthology.orchestra.run.smithi115.stdout:core id : 2 2023-10-09T07:17:28.281 INFO:teuthology.orchestra.run.smithi115.stdout:cpu cores : 4 2023-10-09T07:17:28.282 INFO:teuthology.orchestra.run.smithi115.stdout:apicid : 4 2023-10-09T07:17:28.282 INFO:teuthology.orchestra.run.smithi115.stdout:initial apicid : 4 2023-10-09T07:17:28.282 INFO:teuthology.orchestra.run.smithi115.stdout:fpu : yes 2023-10-09T07:17:28.282 INFO:teuthology.orchestra.run.smithi115.stdout:fpu_exception : yes 2023-10-09T07:17:28.282 INFO:teuthology.orchestra.run.smithi115.stdout:cpuid level : 20 2023-10-09T07:17:28.282 INFO:teuthology.orchestra.run.smithi115.stdout:wp : yes 2023-10-09T07:17:28.282 INFO:teuthology.orchestra.run.smithi115.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single pti intel_ppin tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdt_a rdseed adx smap intel_pt xsaveopt cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts 2023-10-09T07:17:28.282 INFO:teuthology.orchestra.run.smithi115.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa itlb_multihit mmio_stale_data 2023-10-09T07:17:28.282 INFO:teuthology.orchestra.run.smithi115.stdout:bogomips : 7000.35 2023-10-09T07:17:28.282 INFO:teuthology.orchestra.run.smithi115.stdout:clflush size : 64 2023-10-09T07:17:28.282 INFO:teuthology.orchestra.run.smithi115.stdout:cache_alignment : 64 2023-10-09T07:17:28.282 INFO:teuthology.orchestra.run.smithi115.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-09T07:17:28.283 INFO:teuthology.orchestra.run.smithi115.stdout:power management: 2023-10-09T07:17:28.283 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:17:28.283 INFO:teuthology.orchestra.run.smithi115.stdout:processor : 3 2023-10-09T07:17:28.283 INFO:teuthology.orchestra.run.smithi115.stdout:vendor_id : GenuineIntel 2023-10-09T07:17:28.283 INFO:teuthology.orchestra.run.smithi115.stdout:cpu family : 6 2023-10-09T07:17:28.283 INFO:teuthology.orchestra.run.smithi115.stdout:model : 79 2023-10-09T07:17:28.283 INFO:teuthology.orchestra.run.smithi115.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz 2023-10-09T07:17:28.283 INFO:teuthology.orchestra.run.smithi115.stdout:stepping : 1 2023-10-09T07:17:28.283 INFO:teuthology.orchestra.run.smithi115.stdout:microcode : 0xb000010 2023-10-09T07:17:28.284 INFO:teuthology.orchestra.run.smithi115.stdout:cpu MHz : 3531.584 2023-10-09T07:17:28.284 INFO:teuthology.orchestra.run.smithi115.stdout:cache size : 10240 KB 2023-10-09T07:17:28.284 INFO:teuthology.orchestra.run.smithi115.stdout:physical id : 0 2023-10-09T07:17:28.284 INFO:teuthology.orchestra.run.smithi115.stdout:siblings : 8 2023-10-09T07:17:28.284 INFO:teuthology.orchestra.run.smithi115.stdout:core id : 3 2023-10-09T07:17:28.284 INFO:teuthology.orchestra.run.smithi115.stdout:cpu cores : 4 2023-10-09T07:17:28.284 INFO:teuthology.orchestra.run.smithi115.stdout:apicid : 6 2023-10-09T07:17:28.284 INFO:teuthology.orchestra.run.smithi115.stdout:initial apicid : 6 2023-10-09T07:17:28.284 INFO:teuthology.orchestra.run.smithi115.stdout:fpu : yes 2023-10-09T07:17:28.284 INFO:teuthology.orchestra.run.smithi115.stdout:fpu_exception : yes 2023-10-09T07:17:28.284 INFO:teuthology.orchestra.run.smithi115.stdout:cpuid level : 20 2023-10-09T07:17:28.284 INFO:teuthology.orchestra.run.smithi115.stdout:wp : yes 2023-10-09T07:17:28.285 INFO:teuthology.orchestra.run.smithi115.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single pti intel_ppin tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdt_a rdseed adx smap intel_pt xsaveopt cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts 2023-10-09T07:17:28.285 INFO:teuthology.orchestra.run.smithi115.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa itlb_multihit mmio_stale_data 2023-10-09T07:17:28.285 INFO:teuthology.orchestra.run.smithi115.stdout:bogomips : 7000.35 2023-10-09T07:17:28.285 INFO:teuthology.orchestra.run.smithi115.stdout:clflush size : 64 2023-10-09T07:17:28.285 INFO:teuthology.orchestra.run.smithi115.stdout:cache_alignment : 64 2023-10-09T07:17:28.285 INFO:teuthology.orchestra.run.smithi115.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-09T07:17:28.285 INFO:teuthology.orchestra.run.smithi115.stdout:power management: 2023-10-09T07:17:28.285 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:17:28.285 INFO:teuthology.orchestra.run.smithi115.stdout:processor : 4 2023-10-09T07:17:28.285 INFO:teuthology.orchestra.run.smithi115.stdout:vendor_id : GenuineIntel 2023-10-09T07:17:28.285 INFO:teuthology.orchestra.run.smithi115.stdout:cpu family : 6 2023-10-09T07:17:28.285 INFO:teuthology.orchestra.run.smithi115.stdout:model : 79 2023-10-09T07:17:28.286 INFO:teuthology.orchestra.run.smithi115.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz 2023-10-09T07:17:28.286 INFO:teuthology.orchestra.run.smithi115.stdout:stepping : 1 2023-10-09T07:17:28.286 INFO:teuthology.orchestra.run.smithi115.stdout:microcode : 0xb000010 2023-10-09T07:17:28.286 INFO:teuthology.orchestra.run.smithi115.stdout:cpu MHz : 3800.000 2023-10-09T07:17:28.286 INFO:teuthology.orchestra.run.smithi115.stdout:cache size : 10240 KB 2023-10-09T07:17:28.286 INFO:teuthology.orchestra.run.smithi115.stdout:physical id : 0 2023-10-09T07:17:28.286 INFO:teuthology.orchestra.run.smithi115.stdout:siblings : 8 2023-10-09T07:17:28.286 INFO:teuthology.orchestra.run.smithi115.stdout:core id : 0 2023-10-09T07:17:28.286 INFO:teuthology.orchestra.run.smithi115.stdout:cpu cores : 4 2023-10-09T07:17:28.286 INFO:teuthology.orchestra.run.smithi115.stdout:apicid : 1 2023-10-09T07:17:28.286 INFO:teuthology.orchestra.run.smithi115.stdout:initial apicid : 1 2023-10-09T07:17:28.287 INFO:teuthology.orchestra.run.smithi115.stdout:fpu : yes 2023-10-09T07:17:28.287 INFO:teuthology.orchestra.run.smithi115.stdout:fpu_exception : yes 2023-10-09T07:17:28.287 INFO:teuthology.orchestra.run.smithi115.stdout:cpuid level : 20 2023-10-09T07:17:28.287 INFO:teuthology.orchestra.run.smithi115.stdout:wp : yes 2023-10-09T07:17:28.287 INFO:teuthology.orchestra.run.smithi115.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single pti intel_ppin tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdt_a rdseed adx smap intel_pt xsaveopt cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts 2023-10-09T07:17:28.287 INFO:teuthology.orchestra.run.smithi115.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa itlb_multihit mmio_stale_data 2023-10-09T07:17:28.287 INFO:teuthology.orchestra.run.smithi115.stdout:bogomips : 7000.35 2023-10-09T07:17:28.287 INFO:teuthology.orchestra.run.smithi115.stdout:clflush size : 64 2023-10-09T07:17:28.287 INFO:teuthology.orchestra.run.smithi115.stdout:cache_alignment : 64 2023-10-09T07:17:28.287 INFO:teuthology.orchestra.run.smithi115.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-09T07:17:28.287 INFO:teuthology.orchestra.run.smithi115.stdout:power management: 2023-10-09T07:17:28.287 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:17:28.288 INFO:teuthology.orchestra.run.smithi115.stdout:processor : 5 2023-10-09T07:17:28.288 INFO:teuthology.orchestra.run.smithi115.stdout:vendor_id : GenuineIntel 2023-10-09T07:17:28.288 INFO:teuthology.orchestra.run.smithi115.stdout:cpu family : 6 2023-10-09T07:17:28.288 INFO:teuthology.orchestra.run.smithi115.stdout:model : 79 2023-10-09T07:17:28.288 INFO:teuthology.orchestra.run.smithi115.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz 2023-10-09T07:17:28.288 INFO:teuthology.orchestra.run.smithi115.stdout:stepping : 1 2023-10-09T07:17:28.288 INFO:teuthology.orchestra.run.smithi115.stdout:microcode : 0xb000010 2023-10-09T07:17:28.288 INFO:teuthology.orchestra.run.smithi115.stdout:cpu MHz : 3800.000 2023-10-09T07:17:28.288 INFO:teuthology.orchestra.run.smithi115.stdout:cache size : 10240 KB 2023-10-09T07:17:28.288 INFO:teuthology.orchestra.run.smithi115.stdout:physical id : 0 2023-10-09T07:17:28.288 INFO:teuthology.orchestra.run.smithi115.stdout:siblings : 8 2023-10-09T07:17:28.289 INFO:teuthology.orchestra.run.smithi115.stdout:core id : 1 2023-10-09T07:17:28.289 INFO:teuthology.orchestra.run.smithi115.stdout:cpu cores : 4 2023-10-09T07:17:28.289 INFO:teuthology.orchestra.run.smithi115.stdout:apicid : 3 2023-10-09T07:17:28.289 INFO:teuthology.orchestra.run.smithi115.stdout:initial apicid : 3 2023-10-09T07:17:28.289 INFO:teuthology.orchestra.run.smithi115.stdout:fpu : yes 2023-10-09T07:17:28.289 INFO:teuthology.orchestra.run.smithi115.stdout:fpu_exception : yes 2023-10-09T07:17:28.289 INFO:teuthology.orchestra.run.smithi115.stdout:cpuid level : 20 2023-10-09T07:17:28.289 INFO:teuthology.orchestra.run.smithi115.stdout:wp : yes 2023-10-09T07:17:28.289 INFO:teuthology.orchestra.run.smithi115.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single pti intel_ppin tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdt_a rdseed adx smap intel_pt xsaveopt cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts 2023-10-09T07:17:28.289 INFO:teuthology.orchestra.run.smithi115.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa itlb_multihit mmio_stale_data 2023-10-09T07:17:28.289 INFO:teuthology.orchestra.run.smithi115.stdout:bogomips : 7000.35 2023-10-09T07:17:28.289 INFO:teuthology.orchestra.run.smithi115.stdout:clflush size : 64 2023-10-09T07:17:28.290 INFO:teuthology.orchestra.run.smithi115.stdout:cache_alignment : 64 2023-10-09T07:17:28.290 INFO:teuthology.orchestra.run.smithi115.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-09T07:17:28.290 INFO:teuthology.orchestra.run.smithi115.stdout:power management: 2023-10-09T07:17:28.290 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:17:28.290 INFO:teuthology.orchestra.run.smithi115.stdout:processor : 6 2023-10-09T07:17:28.290 INFO:teuthology.orchestra.run.smithi115.stdout:vendor_id : GenuineIntel 2023-10-09T07:17:28.290 INFO:teuthology.orchestra.run.smithi115.stdout:cpu family : 6 2023-10-09T07:17:28.290 INFO:teuthology.orchestra.run.smithi115.stdout:model : 79 2023-10-09T07:17:28.290 INFO:teuthology.orchestra.run.smithi115.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz 2023-10-09T07:17:28.290 INFO:teuthology.orchestra.run.smithi115.stdout:stepping : 1 2023-10-09T07:17:28.290 INFO:teuthology.orchestra.run.smithi115.stdout:microcode : 0xb000010 2023-10-09T07:17:28.290 INFO:teuthology.orchestra.run.smithi115.stdout:cpu MHz : 3800.000 2023-10-09T07:17:28.291 INFO:teuthology.orchestra.run.smithi115.stdout:cache size : 10240 KB 2023-10-09T07:17:28.291 INFO:teuthology.orchestra.run.smithi115.stdout:physical id : 0 2023-10-09T07:17:28.291 INFO:teuthology.orchestra.run.smithi115.stdout:siblings : 8 2023-10-09T07:17:28.291 INFO:teuthology.orchestra.run.smithi115.stdout:core id : 2 2023-10-09T07:17:28.291 INFO:teuthology.orchestra.run.smithi115.stdout:cpu cores : 4 2023-10-09T07:17:28.291 INFO:teuthology.orchestra.run.smithi115.stdout:apicid : 5 2023-10-09T07:17:28.291 INFO:teuthology.orchestra.run.smithi115.stdout:initial apicid : 5 2023-10-09T07:17:28.291 INFO:teuthology.orchestra.run.smithi115.stdout:fpu : yes 2023-10-09T07:17:28.291 INFO:teuthology.orchestra.run.smithi115.stdout:fpu_exception : yes 2023-10-09T07:17:28.291 INFO:teuthology.orchestra.run.smithi115.stdout:cpuid level : 20 2023-10-09T07:17:28.291 INFO:teuthology.orchestra.run.smithi115.stdout:wp : yes 2023-10-09T07:17:28.291 INFO:teuthology.orchestra.run.smithi115.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single pti intel_ppin tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdt_a rdseed adx smap intel_pt xsaveopt cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts 2023-10-09T07:17:28.292 INFO:teuthology.orchestra.run.smithi115.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa itlb_multihit mmio_stale_data 2023-10-09T07:17:28.292 INFO:teuthology.orchestra.run.smithi115.stdout:bogomips : 7000.35 2023-10-09T07:17:28.292 INFO:teuthology.orchestra.run.smithi115.stdout:clflush size : 64 2023-10-09T07:17:28.292 INFO:teuthology.orchestra.run.smithi115.stdout:cache_alignment : 64 2023-10-09T07:17:28.292 INFO:teuthology.orchestra.run.smithi115.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-09T07:17:28.292 INFO:teuthology.orchestra.run.smithi115.stdout:power management: 2023-10-09T07:17:28.292 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:17:28.292 INFO:teuthology.orchestra.run.smithi115.stdout:processor : 7 2023-10-09T07:17:28.292 INFO:teuthology.orchestra.run.smithi115.stdout:vendor_id : GenuineIntel 2023-10-09T07:17:28.292 INFO:teuthology.orchestra.run.smithi115.stdout:cpu family : 6 2023-10-09T07:17:28.292 INFO:teuthology.orchestra.run.smithi115.stdout:model : 79 2023-10-09T07:17:28.292 INFO:teuthology.orchestra.run.smithi115.stdout:model name : Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz 2023-10-09T07:17:28.293 INFO:teuthology.orchestra.run.smithi115.stdout:stepping : 1 2023-10-09T07:17:28.293 INFO:teuthology.orchestra.run.smithi115.stdout:microcode : 0xb000010 2023-10-09T07:17:28.293 INFO:teuthology.orchestra.run.smithi115.stdout:cpu MHz : 1684.691 2023-10-09T07:17:28.293 INFO:teuthology.orchestra.run.smithi115.stdout:cache size : 10240 KB 2023-10-09T07:17:28.293 INFO:teuthology.orchestra.run.smithi115.stdout:physical id : 0 2023-10-09T07:17:28.293 INFO:teuthology.orchestra.run.smithi115.stdout:siblings : 8 2023-10-09T07:17:28.293 INFO:teuthology.orchestra.run.smithi115.stdout:core id : 3 2023-10-09T07:17:28.293 INFO:teuthology.orchestra.run.smithi115.stdout:cpu cores : 4 2023-10-09T07:17:28.293 INFO:teuthology.orchestra.run.smithi115.stdout:apicid : 7 2023-10-09T07:17:28.293 INFO:teuthology.orchestra.run.smithi115.stdout:initial apicid : 7 2023-10-09T07:17:28.293 INFO:teuthology.orchestra.run.smithi115.stdout:fpu : yes 2023-10-09T07:17:28.293 INFO:teuthology.orchestra.run.smithi115.stdout:fpu_exception : yes 2023-10-09T07:17:28.294 INFO:teuthology.orchestra.run.smithi115.stdout:cpuid level : 20 2023-10-09T07:17:28.294 INFO:teuthology.orchestra.run.smithi115.stdout:wp : yes 2023-10-09T07:17:28.294 INFO:teuthology.orchestra.run.smithi115.stdout:flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single pti intel_ppin tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdt_a rdseed adx smap intel_pt xsaveopt cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts 2023-10-09T07:17:28.294 INFO:teuthology.orchestra.run.smithi115.stdout:bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa itlb_multihit mmio_stale_data 2023-10-09T07:17:28.294 INFO:teuthology.orchestra.run.smithi115.stdout:bogomips : 7000.35 2023-10-09T07:17:28.294 INFO:teuthology.orchestra.run.smithi115.stdout:clflush size : 64 2023-10-09T07:17:28.294 INFO:teuthology.orchestra.run.smithi115.stdout:cache_alignment : 64 2023-10-09T07:17:28.294 INFO:teuthology.orchestra.run.smithi115.stdout:address sizes : 46 bits physical, 48 bits virtual 2023-10-09T07:17:28.294 INFO:teuthology.orchestra.run.smithi115.stdout:power management: 2023-10-09T07:17:28.294 INFO:teuthology.orchestra.run.smithi115.stdout: 2023-10-09T07:17:28.294 INFO:teuthology.orchestra.run.smithi115.stdout:<---- s variable is [ ]----> 2023-10-09T07:17:28.295 INFO:teuthology.orchestra.run.smithi115.stderr:++ grep nvmeof.mypool 2023-10-09T07:17:28.295 INFO:teuthology.orchestra.run.smithi115.stderr:++ awk '{print $1}' 2023-10-09T07:17:28.295 INFO:teuthology.orchestra.run.smithi115.stderr:System has not been booted with systemd as init system (PID 1). Can't operate. 2023-10-09T07:17:28.295 INFO:teuthology.orchestra.run.smithi115.stderr:Failed to connect to bus: Host is down 2023-10-09T07:17:28.295 INFO:teuthology.orchestra.run.smithi115.stderr:+ s= 2023-10-09T07:17:28.295 INFO:teuthology.orchestra.run.smithi115.stderr:+ echo '<---- s variable is [ ]---->' 2023-10-09T07:17:28.295 INFO:teuthology.orchestra.run.smithi115.stderr:+ PAGER=cat 2023-10-09T07:17:28.295 INFO:teuthology.orchestra.run.smithi115.stderr:+ journalctl -u 2023-10-09T07:17:28.296 INFO:teuthology.orchestra.run.smithi115.stderr:journalctl: option requires an argument -- 'u' 2023-10-09T07:17:28.554 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:28 smithi115 ceph-mon[108061]: pgmap v131: 32 pgs: 32 active+undersized; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-10-09T07:17:28.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T07:17:28.639 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_barakda_ceph_1cec30186a27ce6ee457e59e165e4c9eb45c6a8d/qa/tasks/cephadm.py", line 1313, in shell _shell(ctx, cluster_name, remote, File "/home/teuthworker/src/github.com_barakda_ceph_1cec30186a27ce6ee457e59e165e4c9eb45c6a8d/qa/tasks/cephadm.py", line 39, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/remote.py", line 522, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi115 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c \'set -ex\n#\ncat /proc/cpuinfo\n# print nvmeof start up logs\ns=$(systemctl list-units | grep nvmeof.mypool | awk \'"\'"\'{print $1}\'"\'"\')\necho "<---- s variable is [ $s ]---->"\nPAGER=cat journalctl -u $s\n\'' 2023-10-09T07:17:28.709 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=1fc1e1a0e9084dabb41adb84c02451bd Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_barakda_ceph_1cec30186a27ce6ee457e59e165e4c9eb45c6a8d/qa/tasks/cephadm.py", line 1313, in shell _shell(ctx, cluster_name, remote, File "/home/teuthworker/src/github.com_barakda_ceph_1cec30186a27ce6ee457e59e165e4c9eb45c6a8d/qa/tasks/cephadm.py", line 39, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/remote.py", line 522, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi115 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c \'set -ex\n#\ncat /proc/cpuinfo\n# print nvmeof start up logs\ns=$(systemctl list-units | grep nvmeof.mypool | awk \'"\'"\'{print $1}\'"\'"\')\necho "<---- s variable is [ $s ]---->"\nPAGER=cat journalctl -u $s\n\'' 2023-10-09T07:17:28.712 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-09T07:17:28.721 INFO:tasks.cephadm:Teardown begin 2023-10-09T07:17:28.721 DEBUG:teuthology.orchestra.run.smithi115:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T07:17:28.755 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-09T07:17:28.755 DEBUG:teuthology.orchestra.run.smithi115:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T07:17:28.813 INFO:tasks.cephadm:Stopping all daemons... 2023-10-09T07:17:28.814 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-09T07:17:28.814 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T07:17:29.212 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:28 smithi115 systemd[1]: Stopping Ceph mon.a for 2817f006-6673-11ee-8db5-212e2dc638e7... 2023-10-09T07:17:29.213 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:29 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mon-a[108057]: 2023-10-09T07:17:29.072+0000 7faa5ff64700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T07:17:29.213 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:29 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mon-a[108057]: 2023-10-09T07:17:29.072+0000 7faa5ff64700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T07:17:29.466 INFO:journalctl@ceph.mon.a.smithi115.stdout:Oct 09 07:17:29 smithi115 bash[132004]: ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T07:17:29.531 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-09T07:17:29.577 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T07:17:29.577 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-09T07:17:29.577 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-09T07:17:29.577 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T07:17:29.808 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:17:29 smithi115 systemd[1]: Stopping Ceph mgr.x for 2817f006-6673-11ee-8db5-212e2dc638e7... 2023-10-09T07:17:30.217 INFO:journalctl@ceph.mgr.x.smithi115.stdout:Oct 09 07:17:29 smithi115 bash[132141]: ceph-2817f006-6673-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T07:17:30.248 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2817f006-6673-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-09T07:17:30.283 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T07:17:30.284 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-09T07:17:30.284 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T07:17:30.284 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-2817f006-6673-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T07:17:30.515 INFO:journalctl@ceph.osd.0.smithi115.stdout:Oct 09 07:17:30 smithi115 systemd[1]: Stopping Ceph osd.0 for 2817f006-6673-11ee-8db5-212e2dc638e7... 2023-10-09T07:17:30.966 INFO:journalctl@ceph.osd.0.smithi115.stdout:Oct 09 07:17:30 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-0[118606]: 2023-10-09T07:17:30.539+0000 7fdeb59cc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T07:17:30.967 INFO:journalctl@ceph.osd.0.smithi115.stdout:Oct 09 07:17:30 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-0[118606]: 2023-10-09T07:17:30.539+0000 7fdeb59cc700 -1 osd.0 19 *** Got signal Terminated *** 2023-10-09T07:17:30.967 INFO:journalctl@ceph.osd.0.smithi115.stdout:Oct 09 07:17:30 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-0[118606]: 2023-10-09T07:17:30.539+0000 7fdeb59cc700 -1 osd.0 19 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T07:17:36.467 INFO:journalctl@ceph.osd.0.smithi115.stdout:Oct 09 07:17:36 smithi115 bash[132276]: ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T07:17:37.799 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2817f006-6673-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T07:17:37.838 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T07:17:37.839 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T07:17:37.839 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T07:17:37.839 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-2817f006-6673-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T07:17:38.077 INFO:journalctl@ceph.osd.1.smithi115.stdout:Oct 09 07:17:37 smithi115 systemd[1]: Stopping Ceph osd.1 for 2817f006-6673-11ee-8db5-212e2dc638e7... 2023-10-09T07:17:38.466 INFO:journalctl@ceph.osd.1.smithi115.stdout:Oct 09 07:17:38 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-1[122785]: 2023-10-09T07:17:38.087+0000 7f235144d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T07:17:38.467 INFO:journalctl@ceph.osd.1.smithi115.stdout:Oct 09 07:17:38 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-1[122785]: 2023-10-09T07:17:38.087+0000 7f235144d700 -1 osd.1 19 *** Got signal Terminated *** 2023-10-09T07:17:38.467 INFO:journalctl@ceph.osd.1.smithi115.stdout:Oct 09 07:17:38 smithi115 ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-1[122785]: 2023-10-09T07:17:38.087+0000 7f235144d700 -1 osd.1 19 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T07:17:44.174 INFO:journalctl@ceph.osd.1.smithi115.stdout:Oct 09 07:17:43 smithi115 bash[132628]: ceph-2817f006-6673-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T07:17:45.066 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2817f006-6673-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T07:17:45.105 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T07:17:45.106 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T07:17:45.106 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T07:17:45.314 INFO:teuthology.orchestra.run.smithi115.stdout:Deleting cluster with fsid: 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:17:46.160 DEBUG:teuthology.orchestra.run.smithi115:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T07:17:46.189 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-09T07:17:46.191 DEBUG:teuthology.misc:Transferring archived files from smithi115:/var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/bdavidov-2023-10-09_06:52:16-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7418388/remote/smithi115/crash 2023-10-09T07:17:46.192 DEBUG:teuthology.orchestra.run.smithi115:> sudo tar c -f - -C /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T07:17:46.261 INFO:teuthology.orchestra.run.smithi115.stderr:tar: /var/lib/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T07:17:46.261 INFO:teuthology.orchestra.run.smithi115.stderr:tar: Error is not recoverable: exiting now 2023-10-09T07:17:46.262 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-09T07:17:46.262 DEBUG:teuthology.orchestra.run.smithi115:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-09T07:17:46.329 INFO:teuthology.orchestra.run.smithi115.stderr:grep: /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-09T07:17:46.330 INFO:tasks.cephadm:Compressing logs... 2023-10-09T07:17:46.330 DEBUG:teuthology.orchestra.run.smithi115:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2023-10-09T07:17:46.397 INFO:teuthology.orchestra.run.smithi115.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2023-10-09T07:17:46.397 INFO:teuthology.orchestra.run.smithi115.stderr:: No such file or directory 2023-10-09T07:17:46.398 INFO:teuthology.orchestra.run.smithi115.stderr:gzip -5 --verbose -- /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-volume.log 2023-10-09T07:17:46.398 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-osd.0.log 2023-10-09T07:17:46.398 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-mgr.x.log 2023-10-09T07:17:46.399 INFO:teuthology.orchestra.run.smithi115.stderr:gzip -5 --verbose -- /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-mon.a.log 2023-10-09T07:17:46.399 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-osd.0.log: /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-osd.1.log 2023-10-09T07:17:46.399 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-mon.a.log: /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-osd.1.log: 91.7% -- replaced with /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-volume.log.gz 2023-10-09T07:17:46.400 INFO:teuthology.orchestra.run.smithi115.stderr: 90.7% -- replaced with /var/log/ceph/cephadm.log.gz 2023-10-09T07:17:46.442 INFO:teuthology.orchestra.run.smithi115.stderr: 89.8% -- replaced with /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-mgr.x.log.gz 2023-10-09T07:17:46.453 INFO:teuthology.orchestra.run.smithi115.stderr: 94.3% -- replaced with /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-osd.0.log.gz 2023-10-09T07:17:46.476 INFO:teuthology.orchestra.run.smithi115.stderr: 94.6% -- replaced with /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-osd.1.log.gz 2023-10-09T07:17:46.482 INFO:teuthology.orchestra.run.smithi115.stderr: 91.2% -- replaced with /var/log/ceph/2817f006-6673-11ee-8db5-212e2dc638e7/ceph-mon.a.log.gz 2023-10-09T07:17:46.484 INFO:teuthology.orchestra.run.smithi115.stderr: 2023-10-09T07:17:46.484 INFO:teuthology.orchestra.run.smithi115.stderr:real 0m0.097s 2023-10-09T07:17:46.484 INFO:teuthology.orchestra.run.smithi115.stderr:user 0m0.264s 2023-10-09T07:17:46.485 INFO:teuthology.orchestra.run.smithi115.stderr:sys 0m0.023s 2023-10-09T07:17:46.485 INFO:tasks.cephadm:Archiving logs... 2023-10-09T07:17:46.485 DEBUG:teuthology.misc:Transferring archived files from smithi115:/var/log/ceph to /home/teuthworker/archive/bdavidov-2023-10-09_06:52:16-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7418388/remote/smithi115/log 2023-10-09T07:17:46.486 DEBUG:teuthology.orchestra.run.smithi115:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T07:17:46.599 INFO:tasks.cephadm:Removing cluster... 2023-10-09T07:17:46.600 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 --force 2023-10-09T07:17:46.790 INFO:teuthology.orchestra.run.smithi115.stdout:Deleting cluster with fsid: 2817f006-6673-11ee-8db5-212e2dc638e7 2023-10-09T07:17:46.938 INFO:tasks.cephadm:Removing cephadm ... 2023-10-09T07:17:46.939 DEBUG:teuthology.orchestra.run.smithi115:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T07:17:46.957 INFO:tasks.cephadm:Teardown complete 2023-10-09T07:17:46.957 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-09T07:17:46.966 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-09T07:17:46.967 DEBUG:teuthology.orchestra.run.smithi115:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T07:17:47.014 INFO:teuthology.orchestra.run.smithi115.stderr:bash: ntpq: command not found 2023-10-09T07:17:47.018 INFO:teuthology.orchestra.run.smithi115.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T07:17:47.018 INFO:teuthology.orchestra.run.smithi115.stdout:=============================================================================== 2023-10-09T07:17:47.018 INFO:teuthology.orchestra.run.smithi115.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 27 -651us[ -645us] +/- 80ms 2023-10-09T07:17:47.018 INFO:teuthology.orchestra.run.smithi115.stdout:^* hv02.front.sepia.ceph.com 3 6 377 27 +222us[ +228us] +/- 77ms 2023-10-09T07:17:47.018 INFO:teuthology.orchestra.run.smithi115.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 27 +341us[ +347us] +/- 63ms 2023-10-09T07:17:47.018 INFO:teuthology.orchestra.run.smithi115.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T07:17:47.019 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-09T07:17:47.028 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-09T07:17:47.029 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-09T07:17:47.037 DEBUG:teuthology.orchestra.run.smithi115:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T07:17:47.151 DEBUG:teuthology.orchestra.run.smithi115:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T07:17:47.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T07:17:47.184 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-09T07:17:47.193 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-09T07:17:47.202 INFO:teuthology.task.internal:Duration was 799.691774 seconds 2023-10-09T07:17:47.202 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-09T07:17:47.210 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-09T07:17:47.210 DEBUG:teuthology.orchestra.run.smithi115:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T07:17:47.270 INFO:teuthology.orchestra.run.smithi115.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T07:17:47.480 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-09T07:17:47.481 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi115.front.sepia.ceph.com 2023-10-09T07:17:47.481 DEBUG:teuthology.orchestra.run.smithi115:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T07:17:47.504 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-09T07:17:47.505 DEBUG:teuthology.orchestra.run.smithi115:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T07:17:47.588 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-09T07:17:47.588 DEBUG:teuthology.orchestra.run.smithi115:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T07:17:47.706 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-09T07:17:47.717 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-09T07:17:47.717 DEBUG:teuthology.orchestra.run.smithi115:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T07:17:47.744 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-09T07:17:47.755 DEBUG:teuthology.orchestra.run.smithi115:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T07:17:47.811 INFO:teuthology.orchestra.run.smithi115.stdout:kernel.core_pattern = core 2023-10-09T07:17:47.836 DEBUG:teuthology.orchestra.run.smithi115:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T07:17:47.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T07:17:47.894 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-09T07:17:47.903 INFO:teuthology.task.internal:Transferring archived files... 2023-10-09T07:17:47.904 DEBUG:teuthology.misc:Transferring archived files from smithi115:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/bdavidov-2023-10-09_06:52:16-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7418388/remote/smithi115 2023-10-09T07:17:47.904 DEBUG:teuthology.orchestra.run.smithi115:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T07:17:47.974 INFO:teuthology.task.internal:Removing archive directory... 2023-10-09T07:17:47.974 DEBUG:teuthology.orchestra.run.smithi115:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T07:17:48.024 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-09T07:17:48.036 INFO:teuthology.task.internal:Not uploading archives. 2023-10-09T07:17:48.036 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-09T07:17:48.047 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-09T07:17:48.047 DEBUG:teuthology.orchestra.run.smithi115:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T07:17:48.082 INFO:teuthology.orchestra.run.smithi115.stdout: 525641 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 9 07:17 /home/ubuntu/cephtest 2023-10-09T07:17:48.083 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-09T07:17:48.093 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-09T07:17:48.114 INFO:teuthology.nuke:Checking targets against current locks 2023-10-09T07:17:48.140 DEBUG:teuthology.nuke:shortname: smithi115 2023-10-09T07:17:48.140 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T07:17:48.162 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi115.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/bdavidov-2023-10-09_06:52:16-rbd:nvmeof-wip-barakda-cephadm-nvmeof-distro-default-smithi/7418388', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 06:55:27.570305', 'locked_by': 'scheduled_bdavidov@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDiUqHywS0xD9nJC4DSbT0+NQ4dCY3SgCS6l903K1h/Xrm6/ERMoYUGRtzWqZyFl9yPJcaHdn6isdZRlEnHlOLw='} 2023-10-09T07:17:48.217 INFO:teuthology.orchestra.console.smithi115:Power off 2023-10-09T07:17:48.217 DEBUG:teuthology.orchestra.console.smithi115:pexpect command: ipmitool -H smithi115.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T07:17:48.243 DEBUG:teuthology.orchestra.console.smithi115:power off output: Chassis Power Control: Down/Off 2023-10-09T07:17:48.243 DEBUG:teuthology.orchestra.console.smithi115:pexpect command: ipmitool -H smithi115.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:17:48.269 DEBUG:teuthology.orchestra.console.smithi115:check power output: Chassis Power is on 2023-10-09T07:17:52.270 DEBUG:teuthology.orchestra.console.smithi115:pexpect command: ipmitool -H smithi115.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:17:52.380 DEBUG:teuthology.orchestra.console.smithi115:check power output: Chassis Power is on 2023-10-09T07:17:56.383 DEBUG:teuthology.orchestra.console.smithi115:pexpect command: ipmitool -H smithi115.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:17:56.493 DEBUG:teuthology.orchestra.console.smithi115:check power output: Chassis Power is on 2023-10-09T07:18:00.497 DEBUG:teuthology.orchestra.console.smithi115:pexpect command: ipmitool -H smithi115.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:18:00.610 DEBUG:teuthology.orchestra.console.smithi115:check power output: Chassis Power is off 2023-10-09T07:18:00.710 INFO:teuthology.orchestra.console.smithi115:Power off completed 2023-10-09T07:18:00.816 INFO:teuthology.run:Summary data: description: rbd:nvmeof/{0-single-container-host base/install cluster/{fixed-3 openstack} workloads/cephadm_nvmeof} duration: 799.6917741298676 failure_reason: 'Command failed on smithi115 with status 1: ''sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bcea8c48517ff154f392584022efa0718f51c9f2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2817f006-6673-11ee-8db5-212e2dc638e7 -- bash -c \''set -ex\n#\ncat /proc/cpuinfo\n# print nvmeof start up logs\ns=$(systemctl list-units | grep nvmeof.mypool | awk \''"\''"\''{print $1}\''"\''"\'')\necho "<---- s variable is [ $s ]---->"\nPAGER=cat journalctl -u $s\n\''''' owner: scheduled_bdavidov@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=1fc1e1a0e9084dabb41adb84c02451bd status: fail success: false 2023-10-09T07:18:00.816 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T07:18:00.898 INFO:teuthology.run:FAIL