2022-03-08T18:25:10.843 INFO:root:teuthology version: 1.1.0-eeaeee38 2022-03-08T18:25:10.847 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --name teuthology-2022-03-06_07:01:04-rados-master-distro-default-smithi --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2022-03-06_07:01:04-rados-master-distro-default-smithi/6724059 --description rados/rook/smoke/{0-distro/ubuntu_20.04 0-kubeadm 0-nvme-loop 1-rook 2-workload/radosbench 3-final cluster/3-node k8s/1.21 net/calico rook/1.7.2} -- /home/teuthworker/archive/teuthology-2022-03-06_07:01:04-rados-master-distro-default-smithi/6724059/orig.config.yaml 2022-03-08T18:25:10.915 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/ 2022-03-08T18:25:10.979 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2022-03-06_07:01:04-rados-master-distro-default-smithi/6724059 branch: master description: rados/rook/smoke/{0-distro/ubuntu_20.04 0-kubeadm 0-nvme-loop 1-rook 2-workload/radosbench 3-final cluster/3-node k8s/1.21 net/calico rook/1.7.2} email: ceph-qa@ceph.io first_in_suite: false job_id: '6724059' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2022-03-06_07:01:04-rados-master-distro-default-smithi nuke-on-error: true os_type: ubuntu os_version: '20.04' overrides: admin_socket: branch: master ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-whitelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 6fe8e680a334072b1ba19b2de006862f98aedbae ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 install: ceph: flavor: default sha1: 6fe8e680a334072b1ba19b2de006862f98aedbae kernel: hwe: true kubeadm: pod_network: calico version: '1.21' rook: rook_branch: v1.7.2 rook_image: rook/ceph:v1.7.2 workunit: branch: master sha1: b8fc4097cbbda97d59dca39eb86991e7f313985b owner: scheduled_teuthology@teuthology priority: 1000 repo: https://github.com/ceph/ceph.git roles: - - host.a - client.a - - host.b - client.b - - host.c - client.c sha1: 6fe8e680a334072b1ba19b2de006862f98aedbae sleep_before_teardown: 0 suite: rados suite_branch: master suite_path: /home/teuthworker/src/github.com_ceph_ceph_b8fc4097cbbda97d59dca39eb86991e7f313985b/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: b8fc4097cbbda97d59dca39eb86991e7f313985b targets: smithi002.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC5wbp2fleVjjiR4hR1suAQJoNsqFic+rolGP2eaUuglv9rPHVz64x/bY83yZrlWP0SGqwaGVCMOhfVXQWp9F6zxCjJIQEZt8ZHKwCCtRPng5rZzE6ZTTKortm2JnNR2Btyx84ZYvuG2J+k/YK5J32kMDv4Z+tXo6+6M+/4O6TgL0YaT1eAyw4BRX1kVFm5PyUL0PKh0J+HE00+8nNdO/CYIZe046CqeAWRsLI1h1yKcm7ARRyaUXMG4BWdzTIuhAv41Yt09m0IKaMZLDjZwNvvoHx8FntQGJcmFSl4mfoauGSDqVRrAq8/Goed0GU+NeSSyRMnbuTEQVTYXMhZXVmZc1TSk6jb7Vud+cWuJUOTAcNF81L13D/cLiGupXwBHzD+Z8zmz4SjDIYUkL6N4HwzLEPaoWGZ6As6HSUO6RL4126mSR0VTJw33Oucgan+PSUfSPYEElQKKf8F8/XZP/5WjebqyRx+MQVDJXmGpLhBGA7ou+eyVUGuEmbWjqul+d0= smithi066.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9mBOyobz24+sAumOcPtvlhwxHfvY/iC6Gkrnf6j+EJf4B9p2jcQnm+w//avtF2EHsiS7AQ6a4kasyiFrFqYzA756eD8b+t37riw1HttYQVMWutQacsbRKQ9CH9uw1zs02meM0VltXQKE9Sit0fzODdhSPHojJTxiPGN3LN3GUNhkYnz2um0tmnhjUdMMP3QXFwGrha6UuWlmJbyh/MIwEePfL5+vs3cXYfwyVssDwmpz99Z7+XrG0rLnlPnhy+6e3o3jQBERhKFpknTZAOU04OJhzxeAVpP/JAXNE8Gg6YGWwKq5lgoCsGrveJCpzgfnRWqtEaqcvuajxu2ZpvUAC/syV2lhzkRTfCp/paWSTvyDC1PY5hAX+ZtJFoz84FPLGEKt8CQwSK13JsQq0lTrXcI9DOS8OVkkeOqLqGWOi9qCw1FQ9DPXrzIgUymtYV2sCIIhe42fgd5N7who8HsnoJRD5xTkaYUZ4GRhSRCfCluUSEpUx0P3uFBe9yi2AGn8= smithi137.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDIrUaErydH07JKbQeupTvdh/nwe4ieGMVeE2mGEY7CVDPWFuKtCf3ACvGUqwblRnWSCmjlYl4Dy3SSjFK7sDxI/2NWHRcOqvyMgjWm2aHBTW+vGfcZpzdTYz44FQ0PHLoU0C8QqC9l1LLJytYZLSFqu6rts0Sb60bt6jigv4pdVirD7tAcLE6lr2NpSgpVRp0o+0egPIra1HfZjv/3dWnFmEXEgmZm9+ieqWiR2eDIFbKNm40UyX6DPdq/K6JiowOjUmXSggEO7EfvNbkR4G/4aZzyMx7Ajd0Nfmv/oFxUkgdESYIXxoBxqDotd8WA6sPZe7B3jTG6R+x2EBcGjZmAyl9l7EyhtTkPn7ivUWsYGBcujCVxpip2RVREupMLiSKZOXf37i+PbfKWc0isxgvPhkbhM/sXsh8hZim+jQyC1ilNiDPwnh5iBw9BUhwaPh6wCg9Nkz5IPqBYB6/+aLmu6D7tAHWAmuGtjn2/QsjxFF65GQfgXniTVRnholEBs3s= tasks: - kubeadm: null - nvme_loop: null - rook: null - rook.shell: - ceph -s - ceph orch status - ceph orch ps - ceph orch ls - ceph orch device ls - install: host.a: null - radosbench: clients: - client.a - rook.shell: commands: - 'ceph orch host label add `hostname` foo ceph orch host ls | grep foo ceph orch host label rm `hostname` foo ceph orch host ls | grep -v foo ' - exec: host.a: - "set -ex\ntoolbox() {\n kubectl -n rook-ceph exec -it deploy/rook-ceph-tools\ \ -- \"$@\"\n}\norig_num_osd=`toolbox ceph osd stat | cut -f3 -d \" \"`\ntoolbox\ \ ceph orch osd rm 0 --force\nremoved_pv=\"\"\nwhile [ \"$removed_pv\" = \"\"\ \ ]\ndo\n removed_pv=`kubectl get pv | grep Released | cut -f1 -d \" \"`\n\ \ sleep 3s\ndone\ntarget_path=`kubectl get pv $removed_pv -o jsonpath='{.spec.local.path}'`\n\ host=`echo $removed_pv | cut -f1 -d \"-\"`\ntoolbox ceph orch device zap $host\ \ $target_path --force\nzap_completion=\"0\"\nwhile [ \"$zap_completion\" =\ \ \"0\" ]\ndo\n zap_completion=`kubectl get job -n rook-ceph rook-ceph-device-zap\ \ -o jsonpath='{.status.succeeded.path}'`\n sleep 3s\ndone\nkubectl patch\ \ pv $removed_pv -p '{\"spec\":{\"claimRef\": null}}'\ntoolbox ceph orch apply\ \ osd --all-available-devices\nkubectl delete job rook-ceph-device-zap -n rook-ceph\n\ num_osd=\"0\"\nwhile [ \"$num_osd\" != \"$orig_num_osd\" ]\ndo\n echo \"\ waiting for osd to come back up\"\n num_osd=`toolbox ceph osd stat | cut\ \ -f3 -d \" \"`\n sleep 30s\ndone\n" - rook.shell: commands: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - ceph orch apply rgw foo - ceph orch apply mds foo - ceph orch apply rbd-mirror - ceph orch apply nfs foo --port 12777 teuthology_branch: master teuthology_sha1: eeaeee38264fad8a01fd1ce912d8908676ed64bc timestamp: 2022-03-06_07:01:04 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.22301 2022-03-08T18:25:10.980 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_b8fc4097cbbda97d59dca39eb86991e7f313985b/qa; will attempt to use it 2022-03-08T18:25:10.980 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_b8fc4097cbbda97d59dca39eb86991e7f313985b/qa/tasks 2022-03-08T18:25:10.981 INFO:teuthology.run_tasks:Running task internal.check_packages... 2022-03-08T18:25:10.983 INFO:teuthology.task.internal:Checking packages... 2022-03-08T18:25:11.007 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash '6fe8e680a334072b1ba19b2de006862f98aedbae' 2022-03-08T18:25:11.008 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-03-08T18:25:11.008 INFO:teuthology.packaging:ref: None 2022-03-08T18:25:11.009 INFO:teuthology.packaging:tag: None 2022-03-08T18:25:11.009 INFO:teuthology.packaging:branch: master 2022-03-08T18:25:11.009 INFO:teuthology.packaging:sha1: 6fe8e680a334072b1ba19b2de006862f98aedbae 2022-03-08T18:25:11.010 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=master 2022-03-08T18:25:11.255 INFO:teuthology.task.internal:Found packages for ceph version 17.0.0-11006-ge98697fd-1focal 2022-03-08T18:25:11.256 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2022-03-08T18:25:11.264 INFO:teuthology.task.internal:no buildpackages task found 2022-03-08T18:25:11.264 INFO:teuthology.run_tasks:Running task internal.save_config... 2022-03-08T18:25:11.278 INFO:teuthology.task.internal:Saving configuration 2022-03-08T18:25:11.292 INFO:teuthology.run_tasks:Running task internal.check_lock... 2022-03-08T18:25:11.299 INFO:teuthology.task.internal.check_lock:Checking locks... 2022-03-08T18:25:11.322 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi002.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2022-03-06_07:01:04-rados-master-distro-default-smithi/6724059', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2022-03-08 18:19:12.025047', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGyfjf5iHS2hmTRtArcneHCt7IdURhXxXBcSnx/SmZmF'} 2022-03-08T18:25:11.344 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi066.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2022-03-06_07:01:04-rados-master-distro-default-smithi/6724059', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2022-03-08 18:19:12.024017', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOHIJJKMlRGr5uTQYr/KsOkpY9ThRQdqISpihvENCXkU'} 2022-03-08T18:25:11.367 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi137.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2022-03-06_07:01:04-rados-master-distro-default-smithi/6724059', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2022-03-08 18:19:12.022697', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN9VanKPZl1deb0ZSg1zoZwrE2WSDJ5nNGRmvuDIEQ7A'} 2022-03-08T18:25:11.367 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2022-03-08T18:25:11.378 INFO:teuthology.task.internal:roles: ubuntu@smithi002.front.sepia.ceph.com - ['host.a', 'client.a'] 2022-03-08T18:25:11.379 INFO:teuthology.task.internal:roles: ubuntu@smithi066.front.sepia.ceph.com - ['host.b', 'client.b'] 2022-03-08T18:25:11.379 INFO:teuthology.task.internal:roles: ubuntu@smithi137.front.sepia.ceph.com - ['host.c', 'client.c'] 2022-03-08T18:25:11.379 INFO:teuthology.run_tasks:Running task console_log... 2022-03-08T18:25:11.484 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f87b32b2ea0>, signals=[15]) 2022-03-08T18:25:11.485 INFO:teuthology.run_tasks:Running task internal.connect... 2022-03-08T18:25:11.496 INFO:teuthology.task.internal:Opening connections... 2022-03-08T18:25:11.496 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi002.front.sepia.ceph.com 2022-03-08T18:25:11.498 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi002.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:25:11.554 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi066.front.sepia.ceph.com 2022-03-08T18:25:11.555 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:25:11.606 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi137.front.sepia.ceph.com 2022-03-08T18:25:11.606 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi137.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:25:11.655 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2022-03-08T18:25:11.666 DEBUG:teuthology.orchestra.run.smithi002:> uname -m 2022-03-08T18:25:11.674 INFO:teuthology.orchestra.run.smithi002.stdout:x86_64 2022-03-08T18:25:11.674 DEBUG:teuthology.orchestra.run.smithi002:> cat /etc/os-release 2022-03-08T18:25:11.723 INFO:teuthology.orchestra.run.smithi002.stdout:NAME="Ubuntu" 2022-03-08T18:25:11.724 INFO:teuthology.orchestra.run.smithi002.stdout:VERSION="20.04.3 LTS (Focal Fossa)" 2022-03-08T18:25:11.724 INFO:teuthology.orchestra.run.smithi002.stdout:ID=ubuntu 2022-03-08T18:25:11.724 INFO:teuthology.orchestra.run.smithi002.stdout:ID_LIKE=debian 2022-03-08T18:25:11.725 INFO:teuthology.orchestra.run.smithi002.stdout:PRETTY_NAME="Ubuntu 20.04.3 LTS" 2022-03-08T18:25:11.725 INFO:teuthology.orchestra.run.smithi002.stdout:VERSION_ID="20.04" 2022-03-08T18:25:11.725 INFO:teuthology.orchestra.run.smithi002.stdout:HOME_URL="https://www.ubuntu.com/" 2022-03-08T18:25:11.725 INFO:teuthology.orchestra.run.smithi002.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2022-03-08T18:25:11.725 INFO:teuthology.orchestra.run.smithi002.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2022-03-08T18:25:11.726 INFO:teuthology.orchestra.run.smithi002.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2022-03-08T18:25:11.726 INFO:teuthology.orchestra.run.smithi002.stdout:VERSION_CODENAME=focal 2022-03-08T18:25:11.726 INFO:teuthology.orchestra.run.smithi002.stdout:UBUNTU_CODENAME=focal 2022-03-08T18:25:11.727 INFO:teuthology.lock.ops:Updating smithi002.front.sepia.ceph.com on lock server 2022-03-08T18:25:11.746 DEBUG:teuthology.orchestra.run.smithi066:> uname -m 2022-03-08T18:25:11.753 INFO:teuthology.orchestra.run.smithi066.stdout:x86_64 2022-03-08T18:25:11.754 DEBUG:teuthology.orchestra.run.smithi066:> cat /etc/os-release 2022-03-08T18:25:11.760 INFO:teuthology.orchestra.run.smithi066.stdout:NAME="Ubuntu" 2022-03-08T18:25:11.761 INFO:teuthology.orchestra.run.smithi066.stdout:VERSION="20.04.3 LTS (Focal Fossa)" 2022-03-08T18:25:11.761 INFO:teuthology.orchestra.run.smithi066.stdout:ID=ubuntu 2022-03-08T18:25:11.761 INFO:teuthology.orchestra.run.smithi066.stdout:ID_LIKE=debian 2022-03-08T18:25:11.762 INFO:teuthology.orchestra.run.smithi066.stdout:PRETTY_NAME="Ubuntu 20.04.3 LTS" 2022-03-08T18:25:11.762 INFO:teuthology.orchestra.run.smithi066.stdout:VERSION_ID="20.04" 2022-03-08T18:25:11.762 INFO:teuthology.orchestra.run.smithi066.stdout:HOME_URL="https://www.ubuntu.com/" 2022-03-08T18:25:11.762 INFO:teuthology.orchestra.run.smithi066.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2022-03-08T18:25:11.763 INFO:teuthology.orchestra.run.smithi066.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2022-03-08T18:25:11.763 INFO:teuthology.orchestra.run.smithi066.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2022-03-08T18:25:11.763 INFO:teuthology.orchestra.run.smithi066.stdout:VERSION_CODENAME=focal 2022-03-08T18:25:11.763 INFO:teuthology.orchestra.run.smithi066.stdout:UBUNTU_CODENAME=focal 2022-03-08T18:25:11.764 INFO:teuthology.lock.ops:Updating smithi066.front.sepia.ceph.com on lock server 2022-03-08T18:25:11.784 DEBUG:teuthology.orchestra.run.smithi137:> uname -m 2022-03-08T18:25:11.792 INFO:teuthology.orchestra.run.smithi137.stdout:x86_64 2022-03-08T18:25:11.792 DEBUG:teuthology.orchestra.run.smithi137:> cat /etc/os-release 2022-03-08T18:25:11.799 INFO:teuthology.orchestra.run.smithi137.stdout:NAME="Ubuntu" 2022-03-08T18:25:11.800 INFO:teuthology.orchestra.run.smithi137.stdout:VERSION="20.04.3 LTS (Focal Fossa)" 2022-03-08T18:25:11.800 INFO:teuthology.orchestra.run.smithi137.stdout:ID=ubuntu 2022-03-08T18:25:11.800 INFO:teuthology.orchestra.run.smithi137.stdout:ID_LIKE=debian 2022-03-08T18:25:11.800 INFO:teuthology.orchestra.run.smithi137.stdout:PRETTY_NAME="Ubuntu 20.04.3 LTS" 2022-03-08T18:25:11.801 INFO:teuthology.orchestra.run.smithi137.stdout:VERSION_ID="20.04" 2022-03-08T18:25:11.801 INFO:teuthology.orchestra.run.smithi137.stdout:HOME_URL="https://www.ubuntu.com/" 2022-03-08T18:25:11.801 INFO:teuthology.orchestra.run.smithi137.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2022-03-08T18:25:11.801 INFO:teuthology.orchestra.run.smithi137.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2022-03-08T18:25:11.802 INFO:teuthology.orchestra.run.smithi137.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2022-03-08T18:25:11.802 INFO:teuthology.orchestra.run.smithi137.stdout:VERSION_CODENAME=focal 2022-03-08T18:25:11.802 INFO:teuthology.orchestra.run.smithi137.stdout:UBUNTU_CODENAME=focal 2022-03-08T18:25:11.802 INFO:teuthology.lock.ops:Updating smithi137.front.sepia.ceph.com on lock server 2022-03-08T18:25:11.819 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2022-03-08T18:25:11.832 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2022-03-08T18:25:11.844 INFO:teuthology.task.internal:Checking for old test directory... 2022-03-08T18:25:11.845 DEBUG:teuthology.orchestra.run.smithi002:> test '!' -e /home/ubuntu/cephtest 2022-03-08T18:25:11.847 DEBUG:teuthology.orchestra.run.smithi066:> test '!' -e /home/ubuntu/cephtest 2022-03-08T18:25:11.850 DEBUG:teuthology.orchestra.run.smithi137:> test '!' -e /home/ubuntu/cephtest 2022-03-08T18:25:11.856 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2022-03-08T18:25:11.866 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2022-03-08T18:25:11.867 DEBUG:teuthology.orchestra.run.smithi002:> test -z $(ls -A /var/lib/ceph) 2022-03-08T18:25:11.869 DEBUG:teuthology.orchestra.run.smithi066:> test -z $(ls -A /var/lib/ceph) 2022-03-08T18:25:11.871 DEBUG:teuthology.orchestra.run.smithi137:> test -z $(ls -A /var/lib/ceph) 2022-03-08T18:25:11.920 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2022-03-08T18:25:12.001 INFO:teuthology.run_tasks:Running task kernel... 2022-03-08T18:25:12.024 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2022-03-08T18:25:12.024 INFO:teuthology.task.kernel:normalize config orig: {'hwe': True} 2022-03-08T18:25:12.024 DEBUG:teuthology.task.kernel:normalized overrides {'host.a': {'hwe': True}, 'client.a': {'hwe': True}, 'host.b': {'hwe': True}, 'client.b': {'hwe': True}, 'host.c': {'hwe': True}, 'client.c': {'hwe': True}} 2022-03-08T18:25:12.025 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro', 'hwe': True}, 'host.b': {'kdb': True, 'sha1': 'distro', 'hwe': True}, 'host.c': {'kdb': True, 'sha1': 'distro', 'hwe': True}}, timeout 300 2022-03-08T18:25:12.026 DEBUG:teuthology.orchestra.run.smithi002:> uname -r 2022-03-08T18:25:12.027 DEBUG:teuthology.orchestra.run.smithi066:> uname -r 2022-03-08T18:25:12.027 DEBUG:teuthology.orchestra.run.smithi137:> uname -r 2022-03-08T18:25:12.034 INFO:teuthology.orchestra.run.smithi066.stdout:5.4.0-97-generic 2022-03-08T18:25:12.035 INFO:teuthology.task.kernel:Running kernel on smithi066: 5.4.0-97-generic 2022-03-08T18:25:12.035 DEBUG:teuthology.orchestra.run.smithi066:> sudo apt-get clean 2022-03-08T18:25:12.039 INFO:teuthology.orchestra.run.smithi137.stdout:5.4.0-97-generic 2022-03-08T18:25:12.039 INFO:teuthology.orchestra.run.smithi002.stdout:5.4.0-97-generic 2022-03-08T18:25:12.040 INFO:teuthology.task.kernel:Running kernel on smithi137: 5.4.0-97-generic 2022-03-08T18:25:12.040 DEBUG:teuthology.orchestra.run.smithi137:> sudo apt-get clean 2022-03-08T18:25:12.040 INFO:teuthology.task.kernel:Running kernel on smithi002: 5.4.0-97-generic 2022-03-08T18:25:12.041 DEBUG:teuthology.orchestra.run.smithi002:> sudo apt-get clean 2022-03-08T18:25:12.154 DEBUG:teuthology.orchestra.run.smithi002:> sudo apt-get update 2022-03-08T18:25:12.164 DEBUG:teuthology.orchestra.run.smithi066:> sudo apt-get update 2022-03-08T18:25:12.197 DEBUG:teuthology.orchestra.run.smithi137:> sudo apt-get update 2022-03-08T18:25:12.325 INFO:teuthology.orchestra.run.smithi066.stdout:Get:1 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2022-03-08T18:25:12.362 INFO:teuthology.orchestra.run.smithi137.stdout:Get:1 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2022-03-08T18:25:12.437 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:25:12.476 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:25:12.480 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:25:12.480 INFO:teuthology.orchestra.run.smithi002.stdout:Get:2 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2022-03-08T18:25:12.519 INFO:teuthology.orchestra.run.smithi066.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2022-03-08T18:25:12.526 INFO:teuthology.orchestra.run.smithi002.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2022-03-08T18:25:12.558 INFO:teuthology.orchestra.run.smithi137.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2022-03-08T18:25:12.643 INFO:teuthology.orchestra.run.smithi066.stdout:Get:4 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [1,278 kB] 2022-03-08T18:25:12.667 INFO:teuthology.orchestra.run.smithi137.stdout:Get:4 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [1,278 kB] 2022-03-08T18:25:12.783 INFO:teuthology.orchestra.run.smithi066.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [390 kB] 2022-03-08T18:25:12.790 INFO:teuthology.orchestra.run.smithi066.stdout:Get:6 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [225 kB] 2022-03-08T18:25:12.795 INFO:teuthology.orchestra.run.smithi066.stdout:Get:7 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [9,732 B] 2022-03-08T18:25:12.796 INFO:teuthology.orchestra.run.smithi066.stdout:Get:8 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [21.7 kB] 2022-03-08T18:25:12.796 INFO:teuthology.orchestra.run.smithi066.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [773 kB] 2022-03-08T18:25:12.816 INFO:teuthology.orchestra.run.smithi066.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [110 kB] 2022-03-08T18:25:12.820 INFO:teuthology.orchestra.run.smithi066.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [504 B] 2022-03-08T18:25:12.820 INFO:teuthology.orchestra.run.smithi066.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [681 kB] 2022-03-08T18:25:12.838 INFO:teuthology.orchestra.run.smithi066.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [536 kB] 2022-03-08T18:25:12.840 INFO:teuthology.orchestra.run.smithi137.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [390 kB] 2022-03-08T18:25:12.847 INFO:teuthology.orchestra.run.smithi137.stdout:Get:6 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [225 kB] 2022-03-08T18:25:12.852 INFO:teuthology.orchestra.run.smithi137.stdout:Get:7 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [9,732 B] 2022-03-08T18:25:12.853 INFO:teuthology.orchestra.run.smithi137.stdout:Get:8 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [773 kB] 2022-03-08T18:25:12.861 INFO:teuthology.orchestra.run.smithi066.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [117 kB] 2022-03-08T18:25:12.865 INFO:teuthology.orchestra.run.smithi066.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [13.2 kB] 2022-03-08T18:25:12.872 INFO:teuthology.orchestra.run.smithi066.stdout:Get:16 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,180 B] 2022-03-08T18:25:12.873 INFO:teuthology.orchestra.run.smithi066.stdout:Get:17 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [20.7 kB] 2022-03-08T18:25:12.874 INFO:teuthology.orchestra.run.smithi066.stdout:Get:18 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,196 B] 2022-03-08T18:25:12.874 INFO:teuthology.orchestra.run.smithi066.stdout:Get:19 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [500 B] 2022-03-08T18:25:12.880 INFO:teuthology.orchestra.run.smithi066.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2022-03-08T18:25:12.885 INFO:teuthology.orchestra.run.smithi137.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [21.7 kB] 2022-03-08T18:25:12.886 INFO:teuthology.orchestra.run.smithi137.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [110 kB] 2022-03-08T18:25:12.888 INFO:teuthology.orchestra.run.smithi137.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [504 B] 2022-03-08T18:25:12.889 INFO:teuthology.orchestra.run.smithi137.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [536 kB] 2022-03-08T18:25:12.890 INFO:teuthology.orchestra.run.smithi002.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2022-03-08T18:25:12.899 INFO:teuthology.orchestra.run.smithi137.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [681 kB] 2022-03-08T18:25:12.913 INFO:teuthology.orchestra.run.smithi137.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [117 kB] 2022-03-08T18:25:12.916 INFO:teuthology.orchestra.run.smithi137.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [13.2 kB] 2022-03-08T18:25:12.939 INFO:teuthology.orchestra.run.smithi137.stdout:Get:16 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [20.7 kB] 2022-03-08T18:25:12.939 INFO:teuthology.orchestra.run.smithi137.stdout:Get:17 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,180 B] 2022-03-08T18:25:12.940 INFO:teuthology.orchestra.run.smithi137.stdout:Get:18 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,196 B] 2022-03-08T18:25:12.940 INFO:teuthology.orchestra.run.smithi137.stdout:Get:19 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [500 B] 2022-03-08T18:25:12.955 INFO:teuthology.orchestra.run.smithi137.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2022-03-08T18:25:12.961 INFO:teuthology.orchestra.run.smithi002.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [390 kB] 2022-03-08T18:25:12.992 INFO:teuthology.orchestra.run.smithi066.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [1,608 kB] 2022-03-08T18:25:13.114 INFO:teuthology.orchestra.run.smithi137.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [1,608 kB] 2022-03-08T18:25:13.115 INFO:teuthology.orchestra.run.smithi002.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [610 kB] 2022-03-08T18:25:13.259 INFO:teuthology.orchestra.run.smithi002.stdout:Get:7 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [1,278 kB] 2022-03-08T18:25:13.396 INFO:teuthology.orchestra.run.smithi066.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [610 kB] 2022-03-08T18:25:13.419 INFO:teuthology.orchestra.run.smithi002.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [1,608 kB] 2022-03-08T18:25:13.433 INFO:teuthology.orchestra.run.smithi066.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [308 kB] 2022-03-08T18:25:13.437 INFO:teuthology.orchestra.run.smithi002.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [225 kB] 2022-03-08T18:25:13.447 INFO:teuthology.orchestra.run.smithi066.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [14.8 kB] 2022-03-08T18:25:13.448 INFO:teuthology.orchestra.run.smithi066.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [23.1 kB] 2022-03-08T18:25:13.449 INFO:teuthology.orchestra.run.smithi066.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [827 kB] 2022-03-08T18:25:13.475 INFO:teuthology.orchestra.run.smithi002.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [9,732 B] 2022-03-08T18:25:13.476 INFO:teuthology.orchestra.run.smithi002.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [773 kB] 2022-03-08T18:25:13.499 INFO:teuthology.orchestra.run.smithi137.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [610 kB] 2022-03-08T18:25:13.511 INFO:teuthology.orchestra.run.smithi066.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [118 kB] 2022-03-08T18:25:13.516 INFO:teuthology.orchestra.run.smithi066.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [500 B] 2022-03-08T18:25:13.517 INFO:teuthology.orchestra.run.smithi066.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [909 kB] 2022-03-08T18:25:13.518 INFO:teuthology.orchestra.run.smithi002.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [21.7 kB] 2022-03-08T18:25:13.519 INFO:teuthology.orchestra.run.smithi002.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [110 kB] 2022-03-08T18:25:13.526 INFO:teuthology.orchestra.run.smithi002.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [504 B] 2022-03-08T18:25:13.526 INFO:teuthology.orchestra.run.smithi002.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [681 kB] 2022-03-08T18:25:13.555 INFO:teuthology.orchestra.run.smithi066.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [672 kB] 2022-03-08T18:25:13.556 INFO:teuthology.orchestra.run.smithi137.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [308 kB] 2022-03-08T18:25:13.579 INFO:teuthology.orchestra.run.smithi137.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [14.8 kB] 2022-03-08T18:25:13.580 INFO:teuthology.orchestra.run.smithi137.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [827 kB] 2022-03-08T18:25:13.588 INFO:teuthology.orchestra.run.smithi066.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [202 kB] 2022-03-08T18:25:13.590 INFO:teuthology.orchestra.run.smithi002.stdout:Get:16 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [536 kB] 2022-03-08T18:25:13.596 INFO:teuthology.orchestra.run.smithi066.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [20.2 kB] 2022-03-08T18:25:13.609 INFO:teuthology.orchestra.run.smithi002.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [308 kB] 2022-03-08T18:25:13.613 INFO:teuthology.orchestra.run.smithi002.stdout:Get:18 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [117 kB] 2022-03-08T18:25:13.617 INFO:teuthology.orchestra.run.smithi002.stdout:Get:19 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [13.2 kB] 2022-03-08T18:25:13.624 INFO:teuthology.orchestra.run.smithi137.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [23.1 kB] 2022-03-08T18:25:13.625 INFO:teuthology.orchestra.run.smithi137.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [118 kB] 2022-03-08T18:25:13.629 INFO:teuthology.orchestra.run.smithi137.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [500 B] 2022-03-08T18:25:13.629 INFO:teuthology.orchestra.run.smithi137.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [909 kB] 2022-03-08T18:25:13.634 INFO:teuthology.orchestra.run.smithi066.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [23.8 kB] 2022-03-08T18:25:13.634 INFO:teuthology.orchestra.run.smithi066.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,464 B] 2022-03-08T18:25:13.635 INFO:teuthology.orchestra.run.smithi066.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,312 B] 2022-03-08T18:25:13.636 INFO:teuthology.orchestra.run.smithi066.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [580 B] 2022-03-08T18:25:13.636 INFO:teuthology.orchestra.run.smithi066.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [42.2 kB] 2022-03-08T18:25:13.654 INFO:teuthology.orchestra.run.smithi002.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [14.8 kB] 2022-03-08T18:25:13.655 INFO:teuthology.orchestra.run.smithi002.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [23.1 kB] 2022-03-08T18:25:13.655 INFO:teuthology.orchestra.run.smithi002.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [827 kB] 2022-03-08T18:25:13.663 INFO:teuthology.orchestra.run.smithi066.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [34.8 kB] 2022-03-08T18:25:13.665 INFO:teuthology.orchestra.run.smithi137.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [672 kB] 2022-03-08T18:25:13.682 INFO:teuthology.orchestra.run.smithi002.stdout:Get:23 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,180 B] 2022-03-08T18:25:13.682 INFO:teuthology.orchestra.run.smithi002.stdout:Get:24 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [20.7 kB] 2022-03-08T18:25:13.683 INFO:teuthology.orchestra.run.smithi002.stdout:Get:25 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,196 B] 2022-03-08T18:25:13.683 INFO:teuthology.orchestra.run.smithi002.stdout:Get:26 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [500 B] 2022-03-08T18:25:13.688 INFO:teuthology.orchestra.run.smithi137.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [202 kB] 2022-03-08T18:25:13.701 INFO:teuthology.orchestra.run.smithi002.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [118 kB] 2022-03-08T18:25:13.704 INFO:teuthology.orchestra.run.smithi002.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [500 B] 2022-03-08T18:25:13.704 INFO:teuthology.orchestra.run.smithi002.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [672 kB] 2022-03-08T18:25:13.718 INFO:teuthology.orchestra.run.smithi137.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [20.2 kB] 2022-03-08T18:25:13.719 INFO:teuthology.orchestra.run.smithi137.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [23.8 kB] 2022-03-08T18:25:13.719 INFO:teuthology.orchestra.run.smithi137.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,464 B] 2022-03-08T18:25:13.739 INFO:teuthology.orchestra.run.smithi002.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [909 kB] 2022-03-08T18:25:13.755 INFO:teuthology.orchestra.run.smithi066.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main Translation-en [10.1 kB] 2022-03-08T18:25:13.756 INFO:teuthology.orchestra.run.smithi066.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [12.9 kB] 2022-03-08T18:25:13.757 INFO:teuthology.orchestra.run.smithi066.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [22.7 kB] 2022-03-08T18:25:13.757 INFO:teuthology.orchestra.run.smithi066.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe Translation-en [15.4 kB] 2022-03-08T18:25:13.758 INFO:teuthology.orchestra.run.smithi066.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [804 B] 2022-03-08T18:25:13.767 INFO:teuthology.orchestra.run.smithi002.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [202 kB] 2022-03-08T18:25:13.770 INFO:teuthology.orchestra.run.smithi002.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [20.2 kB] 2022-03-08T18:25:13.778 INFO:teuthology.orchestra.run.smithi137.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,312 B] 2022-03-08T18:25:13.778 INFO:teuthology.orchestra.run.smithi137.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [580 B] 2022-03-08T18:25:13.779 INFO:teuthology.orchestra.run.smithi137.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [42.2 kB] 2022-03-08T18:25:13.780 INFO:teuthology.orchestra.run.smithi137.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [34.8 kB] 2022-03-08T18:25:13.780 INFO:teuthology.orchestra.run.smithi137.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main Translation-en [10.1 kB] 2022-03-08T18:25:13.788 INFO:teuthology.orchestra.run.smithi002.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [23.8 kB] 2022-03-08T18:25:13.789 INFO:teuthology.orchestra.run.smithi002.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,464 B] 2022-03-08T18:25:13.789 INFO:teuthology.orchestra.run.smithi002.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,312 B] 2022-03-08T18:25:13.790 INFO:teuthology.orchestra.run.smithi002.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [580 B] 2022-03-08T18:25:13.790 INFO:teuthology.orchestra.run.smithi002.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [34.8 kB] 2022-03-08T18:25:13.794 INFO:teuthology.orchestra.run.smithi002.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [42.2 kB] 2022-03-08T18:25:13.801 INFO:teuthology.orchestra.run.smithi137.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [12.9 kB] 2022-03-08T18:25:13.802 INFO:teuthology.orchestra.run.smithi137.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [22.7 kB] 2022-03-08T18:25:13.881 INFO:teuthology.orchestra.run.smithi002.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main Translation-en [10.1 kB] 2022-03-08T18:25:13.881 INFO:teuthology.orchestra.run.smithi002.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [12.9 kB] 2022-03-08T18:25:13.883 INFO:teuthology.orchestra.run.smithi002.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [22.7 kB] 2022-03-08T18:25:13.883 INFO:teuthology.orchestra.run.smithi002.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe Translation-en [15.4 kB] 2022-03-08T18:25:13.884 INFO:teuthology.orchestra.run.smithi002.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [804 B] 2022-03-08T18:25:13.891 INFO:teuthology.orchestra.run.smithi137.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe Translation-en [15.4 kB] 2022-03-08T18:25:13.891 INFO:teuthology.orchestra.run.smithi137.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [804 B] 2022-03-08T18:25:15.969 INFO:teuthology.orchestra.run.smithi066.stdout:Fetched 10.0 MB in 2s (5,897 kB/s) 2022-03-08T18:25:16.154 INFO:teuthology.orchestra.run.smithi137.stdout:Fetched 10.0 MB in 2s (5,443 kB/s) 2022-03-08T18:25:16.671 INFO:teuthology.orchestra.run.smithi002.stdout:Fetched 10.0 MB in 2s (4,460 kB/s) 2022-03-08T18:25:17.137 INFO:teuthology.orchestra.run.smithi066.stdout:Reading package lists... 2022-03-08T18:25:17.162 DEBUG:teuthology.orchestra.run.smithi066:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2022-03-08T18:25:17.237 INFO:teuthology.orchestra.run.smithi066.stdout:Reading package lists... 2022-03-08T18:25:17.346 INFO:teuthology.orchestra.run.smithi137.stdout:Reading package lists... 2022-03-08T18:25:17.371 DEBUG:teuthology.orchestra.run.smithi137:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2022-03-08T18:25:17.414 INFO:teuthology.orchestra.run.smithi066.stdout:Building dependency tree... 2022-03-08T18:25:17.415 INFO:teuthology.orchestra.run.smithi066.stdout:Reading state information... 2022-03-08T18:25:17.441 INFO:teuthology.orchestra.run.smithi137.stdout:Reading package lists... 2022-03-08T18:25:17.570 INFO:teuthology.orchestra.run.smithi066.stdout:The following additional packages will be installed: 2022-03-08T18:25:17.570 INFO:teuthology.orchestra.run.smithi066.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2022-03-08T18:25:17.571 INFO:teuthology.orchestra.run.smithi066.stdout: libusbmuxd6 linux-image-5.13.0-30-generic linux-modules-5.13.0-30-generic 2022-03-08T18:25:17.571 INFO:teuthology.orchestra.run.smithi066.stdout: linux-modules-extra-5.13.0-30-generic thermald upower usbmuxd 2022-03-08T18:25:17.571 INFO:teuthology.orchestra.run.smithi066.stdout:Suggested packages: 2022-03-08T18:25:17.572 INFO:teuthology.orchestra.run.smithi066.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.13-source-5.13.0 2022-03-08T18:25:17.572 INFO:teuthology.orchestra.run.smithi066.stdout: linux-hwe-5.13-tools linux-headers-5.13.0-30-generic 2022-03-08T18:25:17.614 INFO:teuthology.orchestra.run.smithi137.stdout:Building dependency tree... 2022-03-08T18:25:17.614 INFO:teuthology.orchestra.run.smithi137.stdout:Reading state information... 2022-03-08T18:25:17.622 INFO:teuthology.orchestra.run.smithi066.stdout:The following NEW packages will be installed: 2022-03-08T18:25:17.623 INFO:teuthology.orchestra.run.smithi066.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2022-03-08T18:25:17.623 INFO:teuthology.orchestra.run.smithi066.stdout: libusbmuxd6 linux-image-5.13.0-30-generic linux-image-generic-hwe-20.04 2022-03-08T18:25:17.623 INFO:teuthology.orchestra.run.smithi066.stdout: linux-modules-5.13.0-30-generic linux-modules-extra-5.13.0-30-generic 2022-03-08T18:25:17.624 INFO:teuthology.orchestra.run.smithi066.stdout: thermald upower usbmuxd 2022-03-08T18:25:17.755 INFO:teuthology.orchestra.run.smithi137.stdout:The following additional packages will be installed: 2022-03-08T18:25:17.756 INFO:teuthology.orchestra.run.smithi137.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2022-03-08T18:25:17.756 INFO:teuthology.orchestra.run.smithi137.stdout: libusbmuxd6 linux-image-5.13.0-30-generic linux-modules-5.13.0-30-generic 2022-03-08T18:25:17.756 INFO:teuthology.orchestra.run.smithi137.stdout: linux-modules-extra-5.13.0-30-generic thermald upower usbmuxd 2022-03-08T18:25:17.757 INFO:teuthology.orchestra.run.smithi137.stdout:Suggested packages: 2022-03-08T18:25:17.757 INFO:teuthology.orchestra.run.smithi137.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.13-source-5.13.0 2022-03-08T18:25:17.757 INFO:teuthology.orchestra.run.smithi137.stdout: linux-hwe-5.13-tools linux-headers-5.13.0-30-generic 2022-03-08T18:25:17.795 INFO:teuthology.orchestra.run.smithi137.stdout:The following NEW packages will be installed: 2022-03-08T18:25:17.796 INFO:teuthology.orchestra.run.smithi137.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2022-03-08T18:25:17.796 INFO:teuthology.orchestra.run.smithi137.stdout: libusbmuxd6 linux-image-5.13.0-30-generic linux-image-generic-hwe-20.04 2022-03-08T18:25:17.797 INFO:teuthology.orchestra.run.smithi137.stdout: linux-modules-5.13.0-30-generic linux-modules-extra-5.13.0-30-generic 2022-03-08T18:25:17.797 INFO:teuthology.orchestra.run.smithi137.stdout: thermald upower usbmuxd 2022-03-08T18:25:17.810 INFO:teuthology.orchestra.run.smithi066.stdout:0 upgraded, 13 newly installed, 0 to remove and 92 not upgraded. 2022-03-08T18:25:17.811 INFO:teuthology.orchestra.run.smithi066.stdout:Need to get 80.1 MB of archives. 2022-03-08T18:25:17.811 INFO:teuthology.orchestra.run.smithi066.stdout:After this operation, 409 MB of additional disk space will be used. 2022-03-08T18:25:17.811 INFO:teuthology.orchestra.run.smithi066.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2022-03-08T18:25:17.850 INFO:teuthology.orchestra.run.smithi002.stdout:Reading package lists... 2022-03-08T18:25:17.875 DEBUG:teuthology.orchestra.run.smithi002:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2022-03-08T18:25:17.942 INFO:teuthology.orchestra.run.smithi002.stdout:Reading package lists... 2022-03-08T18:25:17.981 INFO:teuthology.orchestra.run.smithi137.stdout:0 upgraded, 13 newly installed, 0 to remove and 92 not upgraded. 2022-03-08T18:25:17.981 INFO:teuthology.orchestra.run.smithi137.stdout:Need to get 80.1 MB of archives. 2022-03-08T18:25:17.981 INFO:teuthology.orchestra.run.smithi137.stdout:After this operation, 409 MB of additional disk space will be used. 2022-03-08T18:25:17.982 INFO:teuthology.orchestra.run.smithi137.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2022-03-08T18:25:18.073 INFO:teuthology.orchestra.run.smithi066.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2022-03-08T18:25:18.078 INFO:teuthology.orchestra.run.smithi066.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2022-03-08T18:25:18.095 INFO:teuthology.orchestra.run.smithi066.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2022-03-08T18:25:18.120 INFO:teuthology.orchestra.run.smithi002.stdout:Building dependency tree... 2022-03-08T18:25:18.120 INFO:teuthology.orchestra.run.smithi002.stdout:Reading state information... 2022-03-08T18:25:18.184 INFO:teuthology.orchestra.run.smithi066.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2022-03-08T18:25:18.198 INFO:teuthology.orchestra.run.smithi066.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.13.0-30-generic amd64 5.13.0-30.33~20.04.1 [18.3 MB] 2022-03-08T18:25:18.241 INFO:teuthology.orchestra.run.smithi137.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2022-03-08T18:25:18.265 INFO:teuthology.orchestra.run.smithi137.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2022-03-08T18:25:18.266 INFO:teuthology.orchestra.run.smithi137.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2022-03-08T18:25:18.275 INFO:teuthology.orchestra.run.smithi002.stdout:The following additional packages will be installed: 2022-03-08T18:25:18.275 INFO:teuthology.orchestra.run.smithi002.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2022-03-08T18:25:18.275 INFO:teuthology.orchestra.run.smithi002.stdout: libusbmuxd6 linux-image-5.13.0-30-generic linux-modules-5.13.0-30-generic 2022-03-08T18:25:18.276 INFO:teuthology.orchestra.run.smithi002.stdout: linux-modules-extra-5.13.0-30-generic thermald upower usbmuxd 2022-03-08T18:25:18.276 INFO:teuthology.orchestra.run.smithi002.stdout:Suggested packages: 2022-03-08T18:25:18.276 INFO:teuthology.orchestra.run.smithi002.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.13-source-5.13.0 2022-03-08T18:25:18.277 INFO:teuthology.orchestra.run.smithi002.stdout: linux-hwe-5.13-tools linux-headers-5.13.0-30-generic 2022-03-08T18:25:18.326 INFO:teuthology.orchestra.run.smithi002.stdout:The following NEW packages will be installed: 2022-03-08T18:25:18.328 INFO:teuthology.orchestra.run.smithi002.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2022-03-08T18:25:18.328 INFO:teuthology.orchestra.run.smithi002.stdout: libusbmuxd6 linux-image-5.13.0-30-generic linux-image-generic-hwe-20.04 2022-03-08T18:25:18.328 INFO:teuthology.orchestra.run.smithi002.stdout: linux-modules-5.13.0-30-generic linux-modules-extra-5.13.0-30-generic 2022-03-08T18:25:18.328 INFO:teuthology.orchestra.run.smithi002.stdout: thermald upower usbmuxd 2022-03-08T18:25:18.353 INFO:teuthology.orchestra.run.smithi137.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2022-03-08T18:25:18.366 INFO:teuthology.orchestra.run.smithi137.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.13.0-30-generic amd64 5.13.0-30.33~20.04.1 [18.3 MB] 2022-03-08T18:25:18.518 INFO:teuthology.orchestra.run.smithi002.stdout:0 upgraded, 13 newly installed, 0 to remove and 92 not upgraded. 2022-03-08T18:25:18.518 INFO:teuthology.orchestra.run.smithi002.stdout:Need to get 80.1 MB of archives. 2022-03-08T18:25:18.518 INFO:teuthology.orchestra.run.smithi002.stdout:After this operation, 409 MB of additional disk space will be used. 2022-03-08T18:25:18.519 INFO:teuthology.orchestra.run.smithi002.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2022-03-08T18:25:18.878 INFO:teuthology.orchestra.run.smithi002.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2022-03-08T18:25:18.980 INFO:teuthology.orchestra.run.smithi002.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2022-03-08T18:25:18.980 INFO:teuthology.orchestra.run.smithi002.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2022-03-08T18:25:18.991 INFO:teuthology.orchestra.run.smithi002.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2022-03-08T18:25:19.061 INFO:teuthology.orchestra.run.smithi002.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.13.0-30-generic amd64 5.13.0-30.33~20.04.1 [18.3 MB] 2022-03-08T18:25:19.462 INFO:teuthology.orchestra.run.smithi066.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.13.0-30-generic amd64 5.13.0-30.33~20.04.1 [10.0 MB] 2022-03-08T18:25:19.869 INFO:teuthology.orchestra.run.smithi137.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.13.0-30-generic amd64 5.13.0-30.33~20.04.1 [10.0 MB] 2022-03-08T18:25:19.921 INFO:teuthology.orchestra.run.smithi066.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.13.0-30-generic amd64 5.13.0-30.33~20.04.1 [51.1 MB] 2022-03-08T18:25:20.392 INFO:teuthology.orchestra.run.smithi137.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.13.0-30-generic amd64 5.13.0-30.33~20.04.1 [51.1 MB] 2022-03-08T18:25:20.456 INFO:teuthology.orchestra.run.smithi002.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.13.0-30-generic amd64 5.13.0-30.33~20.04.1 [10.0 MB] 2022-03-08T18:25:20.802 INFO:teuthology.orchestra.run.smithi002.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.13.0-30-generic amd64 5.13.0-30.33~20.04.1 [51.1 MB] 2022-03-08T18:25:22.754 INFO:teuthology.orchestra.run.smithi066.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.13.0.30.33~20.04.17 [2,616 B] 2022-03-08T18:25:22.754 INFO:teuthology.orchestra.run.smithi066.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.1 [31.6 kB] 2022-03-08T18:25:22.755 INFO:teuthology.orchestra.run.smithi066.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2022-03-08T18:25:22.760 INFO:teuthology.orchestra.run.smithi066.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2022-03-08T18:25:22.763 INFO:teuthology.orchestra.run.smithi066.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2022-03-08T18:25:23.189 INFO:teuthology.orchestra.run.smithi066.stdout:Fetched 80.1 MB in 5s (15.6 MB/s) 2022-03-08T18:25:23.392 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2022-03-08T18:25:23.671 INFO:teuthology.orchestra.run.smithi137.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.13.0.30.33~20.04.17 [2,616 B] 2022-03-08T18:25:23.671 INFO:teuthology.orchestra.run.smithi137.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.1 [31.6 kB] 2022-03-08T18:25:23.672 INFO:teuthology.orchestra.run.smithi137.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2022-03-08T18:25:23.678 INFO:teuthology.orchestra.run.smithi137.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2022-03-08T18:25:23.680 INFO:teuthology.orchestra.run.smithi137.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2022-03-08T18:25:23.820 INFO:teuthology.orchestra.run.smithi002.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.13.0.30.33~20.04.17 [2,616 B] 2022-03-08T18:25:23.820 INFO:teuthology.orchestra.run.smithi002.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.1 [31.6 kB] 2022-03-08T18:25:23.821 INFO:teuthology.orchestra.run.smithi002.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2022-03-08T18:25:23.826 INFO:teuthology.orchestra.run.smithi002.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2022-03-08T18:25:23.829 INFO:teuthology.orchestra.run.smithi002.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2022-03-08T18:25:24.084 INFO:teuthology.orchestra.run.smithi137.stdout:Fetched 80.1 MB in 6s (13.6 MB/s) 2022-03-08T18:25:24.242 INFO:teuthology.orchestra.run.smithi002.stdout:Fetched 80.1 MB in 5s (14.6 MB/s) 2022-03-08T18:25:24.258 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2022-03-08T18:25:24.586 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2022-03-08T18:25:24.857 INFO:teuthology.orchestra.run.smithi066.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 102042 files and directories currently installed.) 2022-03-08T18:25:24.862 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2022-03-08T18:25:25.097 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2022-03-08T18:25:25.450 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libplist3:amd64. 2022-03-08T18:25:25.464 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2022-03-08T18:25:25.508 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2022-03-08T18:25:25.754 INFO:teuthology.orchestra.run.smithi137.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 102042 files and directories currently installed.) 2022-03-08T18:25:25.759 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2022-03-08T18:25:25.803 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2022-03-08T18:25:25.816 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2022-03-08T18:25:25.844 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2022-03-08T18:25:25.996 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2022-03-08T18:25:26.068 INFO:teuthology.orchestra.run.smithi002.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 102042 files and directories currently installed.) 2022-03-08T18:25:26.073 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2022-03-08T18:25:26.156 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2022-03-08T18:25:26.171 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2022-03-08T18:25:26.208 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2022-03-08T18:25:26.213 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2022-03-08T18:25:26.349 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libplist3:amd64. 2022-03-08T18:25:26.363 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2022-03-08T18:25:26.408 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2022-03-08T18:25:26.583 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libupower-glib3:amd64. 2022-03-08T18:25:26.594 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libplist3:amd64. 2022-03-08T18:25:26.598 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2022-03-08T18:25:26.609 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2022-03-08T18:25:26.642 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2022-03-08T18:25:26.652 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2022-03-08T18:25:26.719 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2022-03-08T18:25:26.733 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2022-03-08T18:25:26.810 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2022-03-08T18:25:26.953 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package linux-modules-5.13.0-30-generic. 2022-03-08T18:25:26.956 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2022-03-08T18:25:26.967 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../05-linux-modules-5.13.0-30-generic_5.13.0-30.33~20.04.1_amd64.deb ... 2022-03-08T18:25:26.969 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2022-03-08T18:25:27.005 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2022-03-08T18:25:27.011 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking linux-modules-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:27.138 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2022-03-08T18:25:27.153 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2022-03-08T18:25:27.196 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2022-03-08T18:25:27.367 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2022-03-08T18:25:27.381 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2022-03-08T18:25:27.425 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2022-03-08T18:25:27.533 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libupower-glib3:amd64. 2022-03-08T18:25:27.547 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2022-03-08T18:25:27.583 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2022-03-08T18:25:27.789 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libupower-glib3:amd64. 2022-03-08T18:25:27.801 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2022-03-08T18:25:27.844 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2022-03-08T18:25:27.885 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package linux-modules-5.13.0-30-generic. 2022-03-08T18:25:27.899 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../05-linux-modules-5.13.0-30-generic_5.13.0-30.33~20.04.1_amd64.deb ... 2022-03-08T18:25:27.944 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking linux-modules-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:28.139 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package linux-modules-5.13.0-30-generic. 2022-03-08T18:25:28.151 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../05-linux-modules-5.13.0-30-generic_5.13.0-30.33~20.04.1_amd64.deb ... 2022-03-08T18:25:28.188 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking linux-modules-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:29.303 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package linux-image-5.13.0-30-generic. 2022-03-08T18:25:29.317 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../06-linux-image-5.13.0-30-generic_5.13.0-30.33~20.04.1_amd64.deb ... 2022-03-08T18:25:29.408 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking linux-image-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:29.907 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package linux-modules-extra-5.13.0-30-generic. 2022-03-08T18:25:29.920 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../07-linux-modules-extra-5.13.0-30-generic_5.13.0-30.33~20.04.1_amd64.deb ... 2022-03-08T18:25:29.964 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking linux-modules-extra-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:30.144 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package linux-image-5.13.0-30-generic. 2022-03-08T18:25:30.159 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../06-linux-image-5.13.0-30-generic_5.13.0-30.33~20.04.1_amd64.deb ... 2022-03-08T18:25:30.257 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking linux-image-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:30.406 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package linux-image-5.13.0-30-generic. 2022-03-08T18:25:30.421 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../06-linux-image-5.13.0-30-generic_5.13.0-30.33~20.04.1_amd64.deb ... 2022-03-08T18:25:30.515 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking linux-image-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:30.781 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package linux-modules-extra-5.13.0-30-generic. 2022-03-08T18:25:30.794 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../07-linux-modules-extra-5.13.0-30-generic_5.13.0-30.33~20.04.1_amd64.deb ... 2022-03-08T18:25:30.839 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking linux-modules-extra-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:31.042 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package linux-modules-extra-5.13.0-30-generic. 2022-03-08T18:25:31.057 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../07-linux-modules-extra-5.13.0-30-generic_5.13.0-30.33~20.04.1_amd64.deb ... 2022-03-08T18:25:31.100 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking linux-modules-extra-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:35.707 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2022-03-08T18:25:35.724 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.13.0.30.33~20.04.17_amd64.deb ... 2022-03-08T18:25:35.764 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking linux-image-generic-hwe-20.04 (5.13.0.30.33~20.04.17) ... 2022-03-08T18:25:36.095 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libevdev2:amd64. 2022-03-08T18:25:36.109 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.1_amd64.deb ... 2022-03-08T18:25:36.153 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.1) ... 2022-03-08T18:25:36.482 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package thermald. 2022-03-08T18:25:36.496 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2022-03-08T18:25:36.575 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2022-03-08T18:25:36.800 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2022-03-08T18:25:36.819 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.13.0.30.33~20.04.17_amd64.deb ... 2022-03-08T18:25:36.849 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking linux-image-generic-hwe-20.04 (5.13.0.30.33~20.04.17) ... 2022-03-08T18:25:37.039 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package upower. 2022-03-08T18:25:37.055 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2022-03-08T18:25:37.098 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking upower (0.99.11-1build2) ... 2022-03-08T18:25:37.172 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libevdev2:amd64. 2022-03-08T18:25:37.174 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2022-03-08T18:25:37.188 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.1_amd64.deb ... 2022-03-08T18:25:37.191 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.13.0.30.33~20.04.17_amd64.deb ... 2022-03-08T18:25:37.225 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking linux-image-generic-hwe-20.04 (5.13.0.30.33~20.04.17) ... 2022-03-08T18:25:37.227 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.1) ... 2022-03-08T18:25:37.472 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package usbmuxd. 2022-03-08T18:25:37.489 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2022-03-08T18:25:37.522 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2022-03-08T18:25:37.555 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package thermald. 2022-03-08T18:25:37.568 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2022-03-08T18:25:37.594 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libevdev2:amd64. 2022-03-08T18:25:37.604 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2022-03-08T18:25:37.608 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.1_amd64.deb ... 2022-03-08T18:25:37.652 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.1) ... 2022-03-08T18:25:37.784 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2022-03-08T18:25:37.910 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2022-03-08T18:25:38.005 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package thermald. 2022-03-08T18:25:38.018 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2022-03-08T18:25:38.019 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2022-03-08T18:25:38.088 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2022-03-08T18:25:38.121 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package upower. 2022-03-08T18:25:38.137 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2022-03-08T18:25:38.161 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2022-03-08T18:25:38.189 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking upower (0.99.11-1build2) ... 2022-03-08T18:25:38.297 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2022-03-08T18:25:38.422 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.1) ... 2022-03-08T18:25:38.540 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up upower (0.99.11-1build2) ... 2022-03-08T18:25:38.546 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package usbmuxd. 2022-03-08T18:25:38.560 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2022-03-08T18:25:38.596 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2022-03-08T18:25:38.620 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package upower. 2022-03-08T18:25:38.636 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2022-03-08T18:25:38.688 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking upower (0.99.11-1build2) ... 2022-03-08T18:25:38.921 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2022-03-08T18:25:38.977 INFO:teuthology.orchestra.run.smithi066.stdout:upower.service is a disabled or a static unit, not starting it. 2022-03-08T18:25:39.024 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2022-03-08T18:25:39.042 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2022-03-08T18:25:39.121 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package usbmuxd. 2022-03-08T18:25:39.135 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2022-03-08T18:25:39.168 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2022-03-08T18:25:39.178 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2022-03-08T18:25:39.187 INFO:teuthology.orchestra.run.smithi066.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2022-03-08T18:25:39.196 INFO:teuthology.orchestra.run.smithi066.stdout:Adding system user `usbmux' (UID 114) ... 2022-03-08T18:25:39.196 INFO:teuthology.orchestra.run.smithi066.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2022-03-08T18:25:39.294 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2022-03-08T18:25:39.420 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2022-03-08T18:25:39.459 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2022-03-08T18:25:39.529 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.1) ... 2022-03-08T18:25:39.600 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2022-03-08T18:25:39.647 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up upower (0.99.11-1build2) ... 2022-03-08T18:25:39.658 INFO:teuthology.orchestra.run.smithi066.stdout:Not creating home directory `/var/lib/usbmux'. 2022-03-08T18:25:39.868 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2022-03-08T18:25:39.884 INFO:teuthology.orchestra.run.smithi066.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2022-03-08T18:25:39.921 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2022-03-08T18:25:40.027 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2022-03-08T18:25:40.092 INFO:teuthology.orchestra.run.smithi137.stdout:upower.service is a disabled or a static unit, not starting it. 2022-03-08T18:25:40.153 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2022-03-08T18:25:40.165 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2022-03-08T18:25:40.218 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2022-03-08T18:25:40.219 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2022-03-08T18:25:40.279 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.1) ... 2022-03-08T18:25:40.337 INFO:teuthology.orchestra.run.smithi137.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2022-03-08T18:25:40.344 INFO:teuthology.orchestra.run.smithi137.stdout:Adding system user `usbmux' (UID 114) ... 2022-03-08T18:25:40.344 INFO:teuthology.orchestra.run.smithi137.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2022-03-08T18:25:40.405 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up upower (0.99.11-1build2) ... 2022-03-08T18:25:40.675 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up linux-image-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:40.807 INFO:teuthology.orchestra.run.smithi137.stdout:Not creating home directory `/var/lib/usbmux'. 2022-03-08T18:25:40.873 INFO:teuthology.orchestra.run.smithi002.stdout:upower.service is a disabled or a static unit, not starting it. 2022-03-08T18:25:40.940 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2022-03-08T18:25:41.051 INFO:teuthology.orchestra.run.smithi137.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2022-03-08T18:25:41.120 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2022-03-08T18:25:41.124 INFO:teuthology.orchestra.run.smithi002.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2022-03-08T18:25:41.136 INFO:teuthology.orchestra.run.smithi002.stdout:Adding system user `usbmux' (UID 114) ... 2022-03-08T18:25:41.136 INFO:teuthology.orchestra.run.smithi002.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2022-03-08T18:25:41.431 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2022-03-08T18:25:41.431 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2022-03-08T18:25:41.582 INFO:teuthology.orchestra.run.smithi002.stdout:Not creating home directory `/var/lib/usbmux'. 2022-03-08T18:25:41.808 INFO:teuthology.orchestra.run.smithi002.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2022-03-08T18:25:41.853 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2022-03-08T18:25:41.908 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up linux-image-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:42.002 INFO:teuthology.orchestra.run.smithi066.stdout:I: /boot/vmlinuz is now a symlink to vmlinuz-5.13.0-30-generic 2022-03-08T18:25:42.002 INFO:teuthology.orchestra.run.smithi066.stdout:I: /boot/initrd.img is now a symlink to initrd.img-5.13.0-30-generic 2022-03-08T18:25:42.182 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2022-03-08T18:25:42.182 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2022-03-08T18:25:42.277 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up linux-modules-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:42.683 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up linux-image-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:43.176 INFO:teuthology.orchestra.run.smithi137.stdout:I: /boot/vmlinuz is now a symlink to vmlinuz-5.13.0-30-generic 2022-03-08T18:25:43.177 INFO:teuthology.orchestra.run.smithi137.stdout:I: /boot/initrd.img is now a symlink to initrd.img-5.13.0-30-generic 2022-03-08T18:25:43.326 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up linux-modules-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:43.645 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up linux-modules-extra-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:43.974 INFO:teuthology.orchestra.run.smithi002.stdout:I: /boot/vmlinuz is now a symlink to vmlinuz-5.13.0-30-generic 2022-03-08T18:25:43.975 INFO:teuthology.orchestra.run.smithi002.stdout:I: /boot/initrd.img is now a symlink to initrd.img-5.13.0-30-generic 2022-03-08T18:25:44.142 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up linux-modules-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:44.636 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up linux-modules-extra-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:45.021 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up linux-image-generic-hwe-20.04 (5.13.0.30.33~20.04.17) ... 2022-03-08T18:25:45.155 INFO:teuthology.orchestra.run.smithi066.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.2) ... 2022-03-08T18:25:45.903 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up linux-modules-extra-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:45.929 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up linux-image-generic-hwe-20.04 (5.13.0.30.33~20.04.17) ... 2022-03-08T18:25:46.121 INFO:teuthology.orchestra.run.smithi137.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.2) ... 2022-03-08T18:25:47.254 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up linux-image-generic-hwe-20.04 (5.13.0.30.33~20.04.17) ... 2022-03-08T18:25:47.413 INFO:teuthology.orchestra.run.smithi002.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.2) ... 2022-03-08T18:25:49.903 INFO:teuthology.orchestra.run.smithi066.stdout:Processing triggers for man-db (2.9.1-1) ... 2022-03-08T18:25:50.269 INFO:teuthology.orchestra.run.smithi137.stdout:Processing triggers for man-db (2.9.1-1) ... 2022-03-08T18:25:50.621 INFO:teuthology.orchestra.run.smithi066.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.1) ... 2022-03-08T18:25:50.721 INFO:teuthology.orchestra.run.smithi066.stdout:Processing triggers for linux-image-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:50.915 INFO:teuthology.orchestra.run.smithi066.stdout:/etc/kernel/postinst.d/initramfs-tools: 2022-03-08T18:25:50.915 INFO:teuthology.orchestra.run.smithi066.stdout:update-initramfs: Generating /boot/initrd.img-5.13.0-30-generic 2022-03-08T18:25:50.971 INFO:teuthology.orchestra.run.smithi137.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.1) ... 2022-03-08T18:25:51.071 INFO:teuthology.orchestra.run.smithi137.stdout:Processing triggers for linux-image-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:51.279 INFO:teuthology.orchestra.run.smithi137.stdout:/etc/kernel/postinst.d/initramfs-tools: 2022-03-08T18:25:51.280 INFO:teuthology.orchestra.run.smithi137.stdout:update-initramfs: Generating /boot/initrd.img-5.13.0-30-generic 2022-03-08T18:25:51.835 INFO:teuthology.orchestra.run.smithi002.stdout:Processing triggers for man-db (2.9.1-1) ... 2022-03-08T18:25:52.487 INFO:teuthology.orchestra.run.smithi002.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.1) ... 2022-03-08T18:25:52.596 INFO:teuthology.orchestra.run.smithi002.stdout:Processing triggers for linux-image-5.13.0-30-generic (5.13.0-30.33~20.04.1) ... 2022-03-08T18:25:52.794 INFO:teuthology.orchestra.run.smithi002.stdout:/etc/kernel/postinst.d/initramfs-tools: 2022-03-08T18:25:52.794 INFO:teuthology.orchestra.run.smithi002.stdout:update-initramfs: Generating /boot/initrd.img-5.13.0-30-generic 2022-03-08T18:26:14.587 INFO:teuthology.orchestra.run.smithi137.stdout:/etc/kernel/postinst.d/zz-update-grub: 2022-03-08T18:26:14.588 INFO:teuthology.orchestra.run.smithi137.stdout:Sourcing file `/etc/default/grub' 2022-03-08T18:26:14.600 INFO:teuthology.orchestra.run.smithi137.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2022-03-08T18:26:14.616 INFO:teuthology.orchestra.run.smithi137.stdout:Generating grub configuration file ... 2022-03-08T18:26:14.618 INFO:teuthology.orchestra.run.smithi066.stdout:/etc/kernel/postinst.d/zz-update-grub: 2022-03-08T18:26:14.619 INFO:teuthology.orchestra.run.smithi066.stdout:Sourcing file `/etc/default/grub' 2022-03-08T18:26:14.624 INFO:teuthology.orchestra.run.smithi066.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2022-03-08T18:26:14.647 INFO:teuthology.orchestra.run.smithi066.stdout:Generating grub configuration file ... 2022-03-08T18:26:14.959 INFO:teuthology.orchestra.run.smithi066.stdout:Found linux image: /boot/vmlinuz-5.13.0-30-generic 2022-03-08T18:26:14.988 INFO:teuthology.orchestra.run.smithi137.stdout:Found linux image: /boot/vmlinuz-5.13.0-30-generic 2022-03-08T18:26:14.994 INFO:teuthology.orchestra.run.smithi066.stdout:Found initrd image: /boot/initrd.img-5.13.0-30-generic 2022-03-08T18:26:15.024 INFO:teuthology.orchestra.run.smithi137.stdout:Found initrd image: /boot/initrd.img-5.13.0-30-generic 2022-03-08T18:26:15.373 INFO:teuthology.orchestra.run.smithi066.stdout:Found linux image: /boot/vmlinuz-5.4.0-97-generic 2022-03-08T18:26:15.386 INFO:teuthology.orchestra.run.smithi066.stdout:Found initrd image: /boot/initrd.img-5.4.0-97-generic 2022-03-08T18:26:15.445 INFO:teuthology.orchestra.run.smithi137.stdout:Found linux image: /boot/vmlinuz-5.4.0-97-generic 2022-03-08T18:26:15.457 INFO:teuthology.orchestra.run.smithi137.stdout:Found initrd image: /boot/initrd.img-5.4.0-97-generic 2022-03-08T18:26:16.666 INFO:teuthology.orchestra.run.smithi002.stdout:/etc/kernel/postinst.d/zz-update-grub: 2022-03-08T18:26:16.666 INFO:teuthology.orchestra.run.smithi002.stdout:Sourcing file `/etc/default/grub' 2022-03-08T18:26:16.670 INFO:teuthology.orchestra.run.smithi002.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2022-03-08T18:26:16.694 INFO:teuthology.orchestra.run.smithi002.stdout:Generating grub configuration file ... 2022-03-08T18:26:17.044 INFO:teuthology.orchestra.run.smithi002.stdout:Found linux image: /boot/vmlinuz-5.13.0-30-generic 2022-03-08T18:26:17.068 INFO:teuthology.orchestra.run.smithi002.stdout:Found initrd image: /boot/initrd.img-5.13.0-30-generic 2022-03-08T18:26:17.458 INFO:teuthology.orchestra.run.smithi002.stdout:Found linux image: /boot/vmlinuz-5.4.0-97-generic 2022-03-08T18:26:17.469 INFO:teuthology.orchestra.run.smithi002.stdout:Found initrd image: /boot/initrd.img-5.4.0-97-generic 2022-03-08T18:26:18.015 INFO:teuthology.orchestra.run.smithi066.stdout:done 2022-03-08T18:26:18.083 INFO:teuthology.orchestra.run.smithi137.stdout:done 2022-03-08T18:26:18.351 DEBUG:teuthology.orchestra.run.smithi066:> dpkg -s linux-image-generic-hwe-20.04 2022-03-08T18:26:18.386 INFO:teuthology.orchestra.run.smithi066.stdout:Package: linux-image-generic-hwe-20.04 2022-03-08T18:26:18.386 INFO:teuthology.orchestra.run.smithi066.stdout:Status: install ok installed 2022-03-08T18:26:18.386 INFO:teuthology.orchestra.run.smithi066.stdout:Priority: optional 2022-03-08T18:26:18.387 INFO:teuthology.orchestra.run.smithi066.stdout:Section: kernel 2022-03-08T18:26:18.387 INFO:teuthology.orchestra.run.smithi066.stdout:Installed-Size: 19 2022-03-08T18:26:18.387 INFO:teuthology.orchestra.run.smithi066.stdout:Maintainer: Ubuntu Kernel Team 2022-03-08T18:26:18.387 INFO:teuthology.orchestra.run.smithi066.stdout:Architecture: amd64 2022-03-08T18:26:18.388 INFO:teuthology.orchestra.run.smithi066.stdout:Source: linux-meta-hwe-5.13 2022-03-08T18:26:18.388 INFO:teuthology.orchestra.run.smithi066.stdout:Version: 5.13.0.30.33~20.04.17 2022-03-08T18:26:18.388 INFO:teuthology.orchestra.run.smithi066.stdout:Provides: virtualbox-guest-modules (= 5.13.0-30), wireguard-modules (= 1.0.0), zfs-modules (= 2.0.6-1ubuntu2) 2022-03-08T18:26:18.388 INFO:teuthology.orchestra.run.smithi066.stdout:Depends: linux-image-5.13.0-30-generic, linux-modules-extra-5.13.0-30-generic, linux-firmware, intel-microcode, amd64-microcode 2022-03-08T18:26:18.389 INFO:teuthology.orchestra.run.smithi066.stdout:Recommends: thermald 2022-03-08T18:26:18.389 INFO:teuthology.orchestra.run.smithi066.stdout:Description: Generic Linux kernel image 2022-03-08T18:26:18.389 INFO:teuthology.orchestra.run.smithi066.stdout: This package will always depend on the latest generic kernel image 2022-03-08T18:26:18.389 INFO:teuthology.orchestra.run.smithi066.stdout: available. 2022-03-08T18:26:18.390 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-97-generic Expected: 5.13.0-30-generic 2022-03-08T18:26:18.390 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2022-03-08T18:26:18.390 DEBUG:teuthology.task.kernel:src is distro, skipping download 2022-03-08T18:26:18.391 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2022-03-08T18:26:18.391 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi066.front.sepia.ceph.com, path=None, version=distro) 2022-03-08T18:26:18.391 DEBUG:teuthology.orchestra.run.smithi066:> sudo apt-get clean 2022-03-08T18:26:18.441 DEBUG:teuthology.orchestra.run.smithi137:> dpkg -s linux-image-generic-hwe-20.04 2022-03-08T18:26:18.457 DEBUG:teuthology.orchestra.run.smithi066:> sudo apt-get update 2022-03-08T18:26:18.464 INFO:teuthology.orchestra.run.smithi137.stdout:Package: linux-image-generic-hwe-20.04 2022-03-08T18:26:18.464 INFO:teuthology.orchestra.run.smithi137.stdout:Status: install ok installed 2022-03-08T18:26:18.465 INFO:teuthology.orchestra.run.smithi137.stdout:Priority: optional 2022-03-08T18:26:18.465 INFO:teuthology.orchestra.run.smithi137.stdout:Section: kernel 2022-03-08T18:26:18.465 INFO:teuthology.orchestra.run.smithi137.stdout:Installed-Size: 19 2022-03-08T18:26:18.465 INFO:teuthology.orchestra.run.smithi137.stdout:Maintainer: Ubuntu Kernel Team 2022-03-08T18:26:18.465 INFO:teuthology.orchestra.run.smithi137.stdout:Architecture: amd64 2022-03-08T18:26:18.466 INFO:teuthology.orchestra.run.smithi137.stdout:Source: linux-meta-hwe-5.13 2022-03-08T18:26:18.466 INFO:teuthology.orchestra.run.smithi137.stdout:Version: 5.13.0.30.33~20.04.17 2022-03-08T18:26:18.466 INFO:teuthology.orchestra.run.smithi137.stdout:Provides: virtualbox-guest-modules (= 5.13.0-30), wireguard-modules (= 1.0.0), zfs-modules (= 2.0.6-1ubuntu2) 2022-03-08T18:26:18.466 INFO:teuthology.orchestra.run.smithi137.stdout:Depends: linux-image-5.13.0-30-generic, linux-modules-extra-5.13.0-30-generic, linux-firmware, intel-microcode, amd64-microcode 2022-03-08T18:26:18.467 INFO:teuthology.orchestra.run.smithi137.stdout:Recommends: thermald 2022-03-08T18:26:18.467 INFO:teuthology.orchestra.run.smithi137.stdout:Description: Generic Linux kernel image 2022-03-08T18:26:18.467 INFO:teuthology.orchestra.run.smithi137.stdout: This package will always depend on the latest generic kernel image 2022-03-08T18:26:18.467 INFO:teuthology.orchestra.run.smithi137.stdout: available. 2022-03-08T18:26:18.468 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-97-generic Expected: 5.13.0-30-generic 2022-03-08T18:26:18.468 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2022-03-08T18:26:18.468 DEBUG:teuthology.task.kernel:src is distro, skipping download 2022-03-08T18:26:18.469 INFO:teuthology.task.kernel:Installing distro kernel on host.c... 2022-03-08T18:26:18.469 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi137.front.sepia.ceph.com, path=None, version=distro) 2022-03-08T18:26:18.469 DEBUG:teuthology.orchestra.run.smithi137:> sudo apt-get clean 2022-03-08T18:26:18.525 DEBUG:teuthology.orchestra.run.smithi137:> sudo apt-get update 2022-03-08T18:26:18.616 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2022-03-08T18:26:18.659 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2022-03-08T18:26:18.729 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:26:18.771 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:26:18.811 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2022-03-08T18:26:18.854 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2022-03-08T18:26:18.893 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2022-03-08T18:26:18.940 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2022-03-08T18:26:19.345 INFO:teuthology.orchestra.run.smithi002.stdout:done 2022-03-08T18:26:19.699 DEBUG:teuthology.orchestra.run.smithi002:> dpkg -s linux-image-generic-hwe-20.04 2022-03-08T18:26:19.731 INFO:teuthology.orchestra.run.smithi002.stdout:Package: linux-image-generic-hwe-20.04 2022-03-08T18:26:19.731 INFO:teuthology.orchestra.run.smithi002.stdout:Status: install ok installed 2022-03-08T18:26:19.732 INFO:teuthology.orchestra.run.smithi002.stdout:Priority: optional 2022-03-08T18:26:19.732 INFO:teuthology.orchestra.run.smithi002.stdout:Section: kernel 2022-03-08T18:26:19.732 INFO:teuthology.orchestra.run.smithi002.stdout:Installed-Size: 19 2022-03-08T18:26:19.732 INFO:teuthology.orchestra.run.smithi002.stdout:Maintainer: Ubuntu Kernel Team 2022-03-08T18:26:19.733 INFO:teuthology.orchestra.run.smithi002.stdout:Architecture: amd64 2022-03-08T18:26:19.733 INFO:teuthology.orchestra.run.smithi002.stdout:Source: linux-meta-hwe-5.13 2022-03-08T18:26:19.733 INFO:teuthology.orchestra.run.smithi002.stdout:Version: 5.13.0.30.33~20.04.17 2022-03-08T18:26:19.733 INFO:teuthology.orchestra.run.smithi002.stdout:Provides: virtualbox-guest-modules (= 5.13.0-30), wireguard-modules (= 1.0.0), zfs-modules (= 2.0.6-1ubuntu2) 2022-03-08T18:26:19.733 INFO:teuthology.orchestra.run.smithi002.stdout:Depends: linux-image-5.13.0-30-generic, linux-modules-extra-5.13.0-30-generic, linux-firmware, intel-microcode, amd64-microcode 2022-03-08T18:26:19.734 INFO:teuthology.orchestra.run.smithi002.stdout:Recommends: thermald 2022-03-08T18:26:19.734 INFO:teuthology.orchestra.run.smithi002.stdout:Description: Generic Linux kernel image 2022-03-08T18:26:19.734 INFO:teuthology.orchestra.run.smithi002.stdout: This package will always depend on the latest generic kernel image 2022-03-08T18:26:19.734 INFO:teuthology.orchestra.run.smithi002.stdout: available. 2022-03-08T18:26:19.735 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-97-generic Expected: 5.13.0-30-generic 2022-03-08T18:26:19.735 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2022-03-08T18:26:19.735 DEBUG:teuthology.task.kernel:src is distro, skipping download 2022-03-08T18:26:19.736 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2022-03-08T18:26:19.736 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi002.front.sepia.ceph.com, path=None, version=distro) 2022-03-08T18:26:19.736 DEBUG:teuthology.orchestra.run.smithi002:> sudo apt-get clean 2022-03-08T18:26:19.836 DEBUG:teuthology.orchestra.run.smithi002:> sudo apt-get update 2022-03-08T18:26:19.971 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2022-03-08T18:26:20.083 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:26:20.166 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2022-03-08T18:26:20.185 INFO:teuthology.orchestra.run.smithi066.stdout:Reading package lists... 2022-03-08T18:26:20.208 DEBUG:teuthology.orchestra.run.smithi066:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2022-03-08T18:26:20.216 INFO:teuthology.orchestra.run.smithi137.stdout:Reading package lists... 2022-03-08T18:26:20.240 DEBUG:teuthology.orchestra.run.smithi137:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2022-03-08T18:26:20.248 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2022-03-08T18:26:20.279 INFO:teuthology.orchestra.run.smithi066.stdout:Reading package lists... 2022-03-08T18:26:20.306 INFO:teuthology.orchestra.run.smithi137.stdout:Reading package lists... 2022-03-08T18:26:20.438 INFO:teuthology.orchestra.run.smithi066.stdout:Building dependency tree... 2022-03-08T18:26:20.439 INFO:teuthology.orchestra.run.smithi066.stdout:Reading state information... 2022-03-08T18:26:20.479 INFO:teuthology.orchestra.run.smithi137.stdout:Building dependency tree... 2022-03-08T18:26:20.480 INFO:teuthology.orchestra.run.smithi137.stdout:Reading state information... 2022-03-08T18:26:20.617 INFO:teuthology.orchestra.run.smithi066.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.13.0.30.33~20.04.17). 2022-03-08T18:26:20.617 INFO:teuthology.orchestra.run.smithi066.stdout:0 upgraded, 0 newly installed, 0 to remove and 92 not upgraded. 2022-03-08T18:26:20.619 DEBUG:teuthology.orchestra.run.smithi066:> dpkg -s linux-image-generic-hwe-20.04 2022-03-08T18:26:20.637 INFO:teuthology.orchestra.run.smithi066.stdout:Package: linux-image-generic-hwe-20.04 2022-03-08T18:26:20.637 INFO:teuthology.orchestra.run.smithi066.stdout:Status: install ok installed 2022-03-08T18:26:20.638 INFO:teuthology.orchestra.run.smithi066.stdout:Priority: optional 2022-03-08T18:26:20.638 INFO:teuthology.orchestra.run.smithi066.stdout:Section: kernel 2022-03-08T18:26:20.638 INFO:teuthology.orchestra.run.smithi066.stdout:Installed-Size: 19 2022-03-08T18:26:20.638 INFO:teuthology.orchestra.run.smithi066.stdout:Maintainer: Ubuntu Kernel Team 2022-03-08T18:26:20.639 INFO:teuthology.orchestra.run.smithi066.stdout:Architecture: amd64 2022-03-08T18:26:20.639 INFO:teuthology.orchestra.run.smithi066.stdout:Source: linux-meta-hwe-5.13 2022-03-08T18:26:20.639 INFO:teuthology.orchestra.run.smithi066.stdout:Version: 5.13.0.30.33~20.04.17 2022-03-08T18:26:20.639 INFO:teuthology.orchestra.run.smithi066.stdout:Provides: virtualbox-guest-modules (= 5.13.0-30), wireguard-modules (= 1.0.0), zfs-modules (= 2.0.6-1ubuntu2) 2022-03-08T18:26:20.640 INFO:teuthology.orchestra.run.smithi066.stdout:Depends: linux-image-5.13.0-30-generic, linux-modules-extra-5.13.0-30-generic, linux-firmware, intel-microcode, amd64-microcode 2022-03-08T18:26:20.640 INFO:teuthology.orchestra.run.smithi066.stdout:Recommends: thermald 2022-03-08T18:26:20.640 INFO:teuthology.orchestra.run.smithi066.stdout:Description: Generic Linux kernel image 2022-03-08T18:26:20.640 INFO:teuthology.orchestra.run.smithi066.stdout: This package will always depend on the latest generic kernel image 2022-03-08T18:26:20.641 INFO:teuthology.orchestra.run.smithi066.stdout: available. 2022-03-08T18:26:20.641 DEBUG:teuthology.orchestra.run.smithi066:> mktemp 2022-03-08T18:26:20.657 INFO:teuthology.orchestra.run.smithi137.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.13.0.30.33~20.04.17). 2022-03-08T18:26:20.658 INFO:teuthology.orchestra.run.smithi137.stdout:0 upgraded, 0 newly installed, 0 to remove and 92 not upgraded. 2022-03-08T18:26:20.659 DEBUG:teuthology.orchestra.run.smithi137:> dpkg -s linux-image-generic-hwe-20.04 2022-03-08T18:26:20.679 INFO:teuthology.orchestra.run.smithi137.stdout:Package: linux-image-generic-hwe-20.04 2022-03-08T18:26:20.679 INFO:teuthology.orchestra.run.smithi137.stdout:Status: install ok installed 2022-03-08T18:26:20.680 INFO:teuthology.orchestra.run.smithi137.stdout:Priority: optional 2022-03-08T18:26:20.680 INFO:teuthology.orchestra.run.smithi137.stdout:Section: kernel 2022-03-08T18:26:20.680 INFO:teuthology.orchestra.run.smithi137.stdout:Installed-Size: 19 2022-03-08T18:26:20.680 INFO:teuthology.orchestra.run.smithi137.stdout:Maintainer: Ubuntu Kernel Team 2022-03-08T18:26:20.681 INFO:teuthology.orchestra.run.smithi137.stdout:Architecture: amd64 2022-03-08T18:26:20.681 INFO:teuthology.orchestra.run.smithi137.stdout:Source: linux-meta-hwe-5.13 2022-03-08T18:26:20.681 INFO:teuthology.orchestra.run.smithi137.stdout:Version: 5.13.0.30.33~20.04.17 2022-03-08T18:26:20.681 INFO:teuthology.orchestra.run.smithi137.stdout:Provides: virtualbox-guest-modules (= 5.13.0-30), wireguard-modules (= 1.0.0), zfs-modules (= 2.0.6-1ubuntu2) 2022-03-08T18:26:20.682 INFO:teuthology.orchestra.run.smithi137.stdout:Depends: linux-image-5.13.0-30-generic, linux-modules-extra-5.13.0-30-generic, linux-firmware, intel-microcode, amd64-microcode 2022-03-08T18:26:20.682 INFO:teuthology.orchestra.run.smithi137.stdout:Recommends: thermald 2022-03-08T18:26:20.682 INFO:teuthology.orchestra.run.smithi137.stdout:Description: Generic Linux kernel image 2022-03-08T18:26:20.682 INFO:teuthology.orchestra.run.smithi137.stdout: This package will always depend on the latest generic kernel image 2022-03-08T18:26:20.682 INFO:teuthology.orchestra.run.smithi137.stdout: available. 2022-03-08T18:26:20.683 DEBUG:teuthology.orchestra.run.smithi137:> mktemp 2022-03-08T18:26:20.687 INFO:teuthology.orchestra.run.smithi137.stdout:/tmp/tmp.N4w2DQprOl 2022-03-08T18:26:20.687 DEBUG:teuthology.orchestra.run.smithi137:> sudo cp /boot/grub/grub.cfg /tmp/tmp.N4w2DQprOl 2022-03-08T18:26:20.689 INFO:teuthology.orchestra.run.smithi066.stdout:/tmp/tmp.eUW5EZc1b9 2022-03-08T18:26:20.689 DEBUG:teuthology.orchestra.run.smithi066:> sudo cp /boot/grub/grub.cfg /tmp/tmp.eUW5EZc1b9 2022-03-08T18:26:20.704 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod 0666 /tmp/tmp.eUW5EZc1b9 2022-03-08T18:26:20.744 DEBUG:teuthology.orchestra.run.smithi137:> sudo chmod 0666 /tmp/tmp.N4w2DQprOl 2022-03-08T18:26:20.782 DEBUG:teuthology.orchestra.remote:smithi137:/tmp/tmp.N4w2DQprOl is 8KB 2022-03-08T18:26:20.787 DEBUG:teuthology.orchestra.remote:smithi066:/tmp/tmp.eUW5EZc1b9 is 8KB 2022-03-08T18:26:20.795 DEBUG:teuthology.orchestra.run.smithi137:> rm -fr /tmp/tmp.N4w2DQprOl 2022-03-08T18:26:20.802 DEBUG:teuthology.orchestra.run.smithi137:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2022-03-08T18:26:20.803 DEBUG:teuthology.orchestra.run.smithi066:> rm -fr /tmp/tmp.eUW5EZc1b9 2022-03-08T18:26:20.810 DEBUG:teuthology.orchestra.run.smithi066:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2022-03-08T18:26:20.861 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:26:20.861 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd of=/etc/grub.d/01_ceph_kernel 2022-03-08T18:26:20.861 DEBUG:teuthology.orchestra.run.smithi137:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2022-03-08T18:26:20.865 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:26:20.866 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/grub.d/01_ceph_kernel 2022-03-08T18:26:20.866 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2022-03-08T18:26:20.892 INFO:teuthology.task.kernel:Distro Kernel Version: 5.13.0-30-generic 2022-03-08T18:26:20.892 DEBUG:teuthology.orchestra.run.smithi137:> sudo update-grub 2022-03-08T18:26:20.904 INFO:teuthology.task.kernel:Distro Kernel Version: 5.13.0-30-generic 2022-03-08T18:26:20.904 DEBUG:teuthology.orchestra.run.smithi066:> sudo update-grub 2022-03-08T18:26:21.519 INFO:teuthology.orchestra.run.smithi002.stdout:Reading package lists... 2022-03-08T18:26:21.542 DEBUG:teuthology.orchestra.run.smithi002:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2022-03-08T18:26:21.609 INFO:teuthology.orchestra.run.smithi002.stdout:Reading package lists... 2022-03-08T18:26:21.785 INFO:teuthology.orchestra.run.smithi002.stdout:Building dependency tree... 2022-03-08T18:26:21.786 INFO:teuthology.orchestra.run.smithi002.stdout:Reading state information... 2022-03-08T18:26:21.978 INFO:teuthology.orchestra.run.smithi002.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.13.0.30.33~20.04.17). 2022-03-08T18:26:21.979 INFO:teuthology.orchestra.run.smithi002.stdout:0 upgraded, 0 newly installed, 0 to remove and 92 not upgraded. 2022-03-08T18:26:21.980 DEBUG:teuthology.orchestra.run.smithi002:> dpkg -s linux-image-generic-hwe-20.04 2022-03-08T18:26:22.003 INFO:teuthology.orchestra.run.smithi002.stdout:Package: linux-image-generic-hwe-20.04 2022-03-08T18:26:22.003 INFO:teuthology.orchestra.run.smithi002.stdout:Status: install ok installed 2022-03-08T18:26:22.003 INFO:teuthology.orchestra.run.smithi002.stdout:Priority: optional 2022-03-08T18:26:22.003 INFO:teuthology.orchestra.run.smithi002.stdout:Section: kernel 2022-03-08T18:26:22.004 INFO:teuthology.orchestra.run.smithi002.stdout:Installed-Size: 19 2022-03-08T18:26:22.004 INFO:teuthology.orchestra.run.smithi002.stdout:Maintainer: Ubuntu Kernel Team 2022-03-08T18:26:22.004 INFO:teuthology.orchestra.run.smithi002.stdout:Architecture: amd64 2022-03-08T18:26:22.005 INFO:teuthology.orchestra.run.smithi002.stdout:Source: linux-meta-hwe-5.13 2022-03-08T18:26:22.005 INFO:teuthology.orchestra.run.smithi002.stdout:Version: 5.13.0.30.33~20.04.17 2022-03-08T18:26:22.005 INFO:teuthology.orchestra.run.smithi002.stdout:Provides: virtualbox-guest-modules (= 5.13.0-30), wireguard-modules (= 1.0.0), zfs-modules (= 2.0.6-1ubuntu2) 2022-03-08T18:26:22.005 INFO:teuthology.orchestra.run.smithi002.stdout:Depends: linux-image-5.13.0-30-generic, linux-modules-extra-5.13.0-30-generic, linux-firmware, intel-microcode, amd64-microcode 2022-03-08T18:26:22.006 INFO:teuthology.orchestra.run.smithi002.stdout:Recommends: thermald 2022-03-08T18:26:22.006 INFO:teuthology.orchestra.run.smithi002.stdout:Description: Generic Linux kernel image 2022-03-08T18:26:22.006 INFO:teuthology.orchestra.run.smithi002.stdout: This package will always depend on the latest generic kernel image 2022-03-08T18:26:22.006 INFO:teuthology.orchestra.run.smithi002.stdout: available. 2022-03-08T18:26:22.007 DEBUG:teuthology.orchestra.run.smithi002:> mktemp 2022-03-08T18:26:22.009 INFO:teuthology.orchestra.run.smithi066.stderr:Sourcing file `/etc/default/grub' 2022-03-08T18:26:22.010 INFO:teuthology.orchestra.run.smithi066.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2022-03-08T18:26:22.013 INFO:teuthology.orchestra.run.smithi137.stderr:Sourcing file `/etc/default/grub' 2022-03-08T18:26:22.016 INFO:teuthology.orchestra.run.smithi137.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2022-03-08T18:26:22.021 INFO:teuthology.orchestra.run.smithi066.stderr:Generating grub configuration file ... 2022-03-08T18:26:22.031 INFO:teuthology.orchestra.run.smithi137.stderr:Generating grub configuration file ... 2022-03-08T18:26:22.051 INFO:teuthology.orchestra.run.smithi002.stdout:/tmp/tmp.lta7BAHOTN 2022-03-08T18:26:22.051 DEBUG:teuthology.orchestra.run.smithi002:> sudo cp /boot/grub/grub.cfg /tmp/tmp.lta7BAHOTN 2022-03-08T18:26:22.067 DEBUG:teuthology.orchestra.run.smithi002:> sudo chmod 0666 /tmp/tmp.lta7BAHOTN 2022-03-08T18:26:22.150 DEBUG:teuthology.orchestra.remote:smithi002:/tmp/tmp.lta7BAHOTN is 8KB 2022-03-08T18:26:22.164 DEBUG:teuthology.orchestra.run.smithi002:> rm -fr /tmp/tmp.lta7BAHOTN 2022-03-08T18:26:22.171 DEBUG:teuthology.orchestra.run.smithi002:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2022-03-08T18:26:22.228 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:26:22.228 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd of=/etc/grub.d/01_ceph_kernel 2022-03-08T18:26:22.228 DEBUG:teuthology.orchestra.run.smithi002:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2022-03-08T18:26:22.260 INFO:teuthology.task.kernel:Distro Kernel Version: 5.13.0-30-generic 2022-03-08T18:26:22.261 DEBUG:teuthology.orchestra.run.smithi002:> sudo update-grub 2022-03-08T18:26:22.285 INFO:teuthology.orchestra.run.smithi066.stderr:Found linux image: /boot/vmlinuz-5.13.0-30-generic 2022-03-08T18:26:22.297 INFO:teuthology.orchestra.run.smithi066.stderr:Found initrd image: /boot/initrd.img-5.13.0-30-generic 2022-03-08T18:26:22.327 INFO:teuthology.orchestra.run.smithi137.stderr:Found linux image: /boot/vmlinuz-5.13.0-30-generic 2022-03-08T18:26:22.339 INFO:teuthology.orchestra.run.smithi137.stderr:Found initrd image: /boot/initrd.img-5.13.0-30-generic 2022-03-08T18:26:22.641 INFO:teuthology.orchestra.run.smithi066.stderr:Found linux image: /boot/vmlinuz-5.4.0-97-generic 2022-03-08T18:26:22.653 INFO:teuthology.orchestra.run.smithi066.stderr:Found initrd image: /boot/initrd.img-5.4.0-97-generic 2022-03-08T18:26:22.736 INFO:teuthology.orchestra.run.smithi137.stderr:Found linux image: /boot/vmlinuz-5.4.0-97-generic 2022-03-08T18:26:22.748 INFO:teuthology.orchestra.run.smithi137.stderr:Found initrd image: /boot/initrd.img-5.4.0-97-generic 2022-03-08T18:26:23.448 INFO:teuthology.orchestra.run.smithi002.stderr:Sourcing file `/etc/default/grub' 2022-03-08T18:26:23.457 INFO:teuthology.orchestra.run.smithi002.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2022-03-08T18:26:23.468 INFO:teuthology.orchestra.run.smithi002.stderr:Generating grub configuration file ... 2022-03-08T18:26:23.689 INFO:teuthology.orchestra.run.smithi002.stderr:Found linux image: /boot/vmlinuz-5.13.0-30-generic 2022-03-08T18:26:23.701 INFO:teuthology.orchestra.run.smithi002.stderr:Found initrd image: /boot/initrd.img-5.13.0-30-generic 2022-03-08T18:26:24.090 INFO:teuthology.orchestra.run.smithi002.stderr:Found linux image: /boot/vmlinuz-5.4.0-97-generic 2022-03-08T18:26:24.101 INFO:teuthology.orchestra.run.smithi002.stderr:Found initrd image: /boot/initrd.img-5.4.0-97-generic 2022-03-08T18:26:24.286 INFO:teuthology.orchestra.run.smithi066.stderr:done 2022-03-08T18:26:24.288 DEBUG:teuthology.orchestra.run.smithi066:> sudo shutdown -r now 2022-03-08T18:26:24.354 INFO:teuthology.orchestra.run.smithi137.stderr:done 2022-03-08T18:26:24.356 DEBUG:teuthology.orchestra.run.smithi137:> sudo shutdown -r now 2022-03-08T18:26:25.039 INFO:teuthology.orchestra.run.smithi002.stderr:done 2022-03-08T18:26:25.040 DEBUG:teuthology.orchestra.run.smithi002:> sudo shutdown -r now 2022-03-08T18:26:54.318 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2022-03-08T18:26:54.319 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2022-03-08T18:26:54.320 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:26:54.359 INFO:teuthology.task.kernel:Checking client host.c for new kernel version... 2022-03-08T18:26:54.359 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi137.front.sepia.ceph.com' 2022-03-08T18:26:54.360 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi137.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:26:55.044 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2022-03-08T18:26:55.044 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi002.front.sepia.ceph.com' 2022-03-08T18:26:55.045 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi002.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:27:12.715 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.66 2022-03-08T18:27:12.719 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.137 2022-03-08T18:27:13.486 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.2 2022-03-08T18:27:42.742 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2022-03-08T18:27:42.743 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:27:42.745 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi137.front.sepia.ceph.com' 2022-03-08T18:27:42.745 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi137.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:27:43.489 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi002.front.sepia.ceph.com' 2022-03-08T18:27:43.490 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi002.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:27:45.803 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.66 2022-03-08T18:27:59.424 DEBUG:teuthology.orchestra.run.smithi002:> true 2022-03-08T18:28:00.038 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi002.front.sepia.ceph.com' 2022-03-08T18:28:00.039 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.13.0-30-generic"... 2022-03-08T18:28:00.039 DEBUG:teuthology.orchestra.run.smithi002:> uname -r 2022-03-08T18:28:00.046 INFO:teuthology.orchestra.run.smithi002.stdout:5.13.0-30-generic 2022-03-08T18:28:00.047 DEBUG:teuthology.task.kernel:current kernel version is 5.13.0-30-generic vs 5.13.0-30-generic 2022-03-08T18:28:00.047 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2022-03-08T18:28:00.047 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2022-03-08T18:28:01.049 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2022-03-08T18:28:01.049 DEBUG:teuthology.orchestra.run.smithi002:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2022-03-08T18:28:01.098 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.137 2022-03-08T18:28:01.112 INFO:teuthology.orchestra.run.smithi002.stdout:ttyS1 2022-03-08T18:28:01.153 DEBUG:teuthology.parallel:result is None 2022-03-08T18:28:15.804 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2022-03-08T18:28:15.805 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:28:18.827 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.66 2022-03-08T18:28:31.099 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi137.front.sepia.ceph.com' 2022-03-08T18:28:31.100 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi137.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:28:31.538 DEBUG:teuthology.orchestra.run.smithi137:> true 2022-03-08T18:28:32.053 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi137.front.sepia.ceph.com' 2022-03-08T18:28:32.053 INFO:teuthology.task.kernel:Checking kernel version of host.c, want "5.13.0-30-generic"... 2022-03-08T18:28:32.053 DEBUG:teuthology.orchestra.run.smithi137:> uname -r 2022-03-08T18:28:32.058 INFO:teuthology.orchestra.run.smithi137.stdout:5.13.0-30-generic 2022-03-08T18:28:32.059 DEBUG:teuthology.task.kernel:current kernel version is 5.13.0-30-generic vs 5.13.0-30-generic 2022-03-08T18:28:32.059 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2022-03-08T18:28:32.060 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2022-03-08T18:28:33.061 INFO:teuthology.task.kernel:Enabling kdb on host.c... 2022-03-08T18:28:33.062 DEBUG:teuthology.orchestra.run.smithi137:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2022-03-08T18:28:33.133 INFO:teuthology.orchestra.run.smithi137.stdout:ttyS1 2022-03-08T18:28:33.156 DEBUG:teuthology.parallel:result is None 2022-03-08T18:28:48.829 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2022-03-08T18:28:48.829 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:28:49.036 DEBUG:teuthology.orchestra.run.smithi066:> true 2022-03-08T18:28:49.532 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi066.front.sepia.ceph.com' 2022-03-08T18:28:49.533 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.13.0-30-generic"... 2022-03-08T18:28:49.533 DEBUG:teuthology.orchestra.run.smithi066:> uname -r 2022-03-08T18:28:49.540 INFO:teuthology.orchestra.run.smithi066.stdout:5.13.0-30-generic 2022-03-08T18:28:49.541 DEBUG:teuthology.task.kernel:current kernel version is 5.13.0-30-generic vs 5.13.0-30-generic 2022-03-08T18:28:49.541 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2022-03-08T18:28:49.541 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2022-03-08T18:28:50.542 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2022-03-08T18:28:50.542 DEBUG:teuthology.orchestra.run.smithi066:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2022-03-08T18:28:50.622 INFO:teuthology.orchestra.run.smithi066.stdout:ttyS1 2022-03-08T18:28:50.647 DEBUG:teuthology.parallel:result is None 2022-03-08T18:28:50.648 INFO:teuthology.run_tasks:Running task internal.base... 2022-03-08T18:28:50.659 INFO:teuthology.task.internal:Creating test directory... 2022-03-08T18:28:50.660 DEBUG:teuthology.orchestra.run.smithi002:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2022-03-08T18:28:50.663 DEBUG:teuthology.orchestra.run.smithi066:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2022-03-08T18:28:50.665 DEBUG:teuthology.orchestra.run.smithi137:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2022-03-08T18:28:50.673 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2022-03-08T18:28:50.684 INFO:teuthology.run_tasks:Running task internal.archive... 2022-03-08T18:28:50.696 INFO:teuthology.task.internal:Creating archive directory... 2022-03-08T18:28:50.696 DEBUG:teuthology.orchestra.run.smithi002:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2022-03-08T18:28:50.710 DEBUG:teuthology.orchestra.run.smithi066:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2022-03-08T18:28:50.713 DEBUG:teuthology.orchestra.run.smithi137:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2022-03-08T18:28:50.737 INFO:teuthology.run_tasks:Running task internal.coredump... 2022-03-08T18:28:50.748 INFO:teuthology.task.internal:Enabling coredump saving... 2022-03-08T18:28:50.748 DEBUG:teuthology.orchestra.run.smithi002:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2022-03-08T18:28:50.751 DEBUG:teuthology.orchestra.run.smithi066:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2022-03-08T18:28:50.772 INFO:teuthology.orchestra.run.smithi002.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-03-08T18:28:50.775 DEBUG:teuthology.orchestra.run.smithi137:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2022-03-08T18:28:50.783 INFO:teuthology.orchestra.run.smithi002.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-03-08T18:28:50.797 INFO:teuthology.orchestra.run.smithi137.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-03-08T18:28:50.799 INFO:teuthology.orchestra.run.smithi066.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-03-08T18:28:50.808 INFO:teuthology.orchestra.run.smithi137.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-03-08T18:28:50.811 INFO:teuthology.orchestra.run.smithi066.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-03-08T18:28:50.812 INFO:teuthology.run_tasks:Running task internal.sudo... 2022-03-08T18:28:50.823 INFO:teuthology.task.internal:Configuring sudo... 2022-03-08T18:28:50.824 DEBUG:teuthology.orchestra.run.smithi002:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2022-03-08T18:28:50.830 DEBUG:teuthology.orchestra.run.smithi066:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2022-03-08T18:28:50.833 DEBUG:teuthology.orchestra.run.smithi137:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2022-03-08T18:28:50.850 INFO:teuthology.run_tasks:Running task internal.syslog... 2022-03-08T18:28:50.863 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2022-03-08T18:28:50.864 DEBUG:teuthology.orchestra.run.smithi002:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2022-03-08T18:28:50.866 DEBUG:teuthology.orchestra.run.smithi066:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2022-03-08T18:28:50.891 DEBUG:teuthology.orchestra.run.smithi137:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2022-03-08T18:28:50.902 DEBUG:teuthology.orchestra.run.smithi002:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2022-03-08T18:28:50.910 DEBUG:teuthology.orchestra.run.smithi002:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2022-03-08T18:28:50.959 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:28:50.960 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2022-03-08T18:28:50.978 DEBUG:teuthology.orchestra.run.smithi066:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2022-03-08T18:28:50.987 DEBUG:teuthology.orchestra.run.smithi066:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2022-03-08T18:28:51.036 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:28:51.037 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2022-03-08T18:28:51.058 DEBUG:teuthology.orchestra.run.smithi137:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2022-03-08T18:28:51.067 DEBUG:teuthology.orchestra.run.smithi137:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2022-03-08T18:28:51.072 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:28:51.073 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2022-03-08T18:28:51.092 DEBUG:teuthology.orchestra.run.smithi002:> sudo service rsyslog restart 2022-03-08T18:28:51.095 DEBUG:teuthology.orchestra.run.smithi066:> sudo service rsyslog restart 2022-03-08T18:28:51.098 DEBUG:teuthology.orchestra.run.smithi137:> sudo service rsyslog restart 2022-03-08T18:28:51.217 INFO:teuthology.run_tasks:Running task internal.timer... 2022-03-08T18:28:51.227 INFO:teuthology.task.internal:Starting timer... 2022-03-08T18:28:51.227 INFO:teuthology.run_tasks:Running task pcp... 2022-03-08T18:28:51.241 INFO:teuthology.run_tasks:Running task selinux... 2022-03-08T18:28:51.255 DEBUG:teuthology.task.selinux:Excluding smithi002: OS 'ubuntu' does not support SELinux 2022-03-08T18:28:51.255 DEBUG:teuthology.task.selinux:Excluding smithi066: OS 'ubuntu' does not support SELinux 2022-03-08T18:28:51.256 DEBUG:teuthology.task.selinux:Excluding smithi137: OS 'ubuntu' does not support SELinux 2022-03-08T18:28:51.256 DEBUG:teuthology.task.selinux:Getting current SELinux state 2022-03-08T18:28:51.257 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2022-03-08T18:28:51.257 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2022-03-08T18:28:51.257 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2022-03-08T18:28:51.275 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master was just updated or references a specific commit; assuming it is current 2022-03-08T18:28:51.276 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master to origin/master 2022-03-08T18:28:51.299 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2022-03-08T18:28:51.303 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi002.front.sepia.ceph.com,smithi066.front.sepia.ceph.com,smithi137.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/cephlab.yml 2022-03-08T18:31:57.210 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi002.front.sepia.ceph.com'), Remote(name='ubuntu@smithi066.front.sepia.ceph.com'), Remote(name='ubuntu@smithi137.front.sepia.ceph.com')] 2022-03-08T18:31:57.212 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi002.front.sepia.ceph.com' 2022-03-08T18:31:57.214 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi002.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:31:57.270 DEBUG:teuthology.orchestra.run.smithi002:> true 2022-03-08T18:31:57.396 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2022-03-08T18:31:57.397 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:31:57.443 DEBUG:teuthology.orchestra.run.smithi066:> true 2022-03-08T18:31:57.569 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi137.front.sepia.ceph.com' 2022-03-08T18:31:57.569 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi137.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-03-08T18:31:57.623 DEBUG:teuthology.orchestra.run.smithi137:> true 2022-03-08T18:31:57.744 INFO:teuthology.run_tasks:Running task clock... 2022-03-08T18:31:57.758 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2022-03-08T18:31:57.759 INFO:teuthology.orchestra.run:Running command with timeout 360 2022-03-08T18:31:57.761 DEBUG:teuthology.orchestra.run.smithi002:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2022-03-08T18:31:57.763 INFO:teuthology.orchestra.run:Running command with timeout 360 2022-03-08T18:31:57.764 DEBUG:teuthology.orchestra.run.smithi066:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2022-03-08T18:31:57.766 INFO:teuthology.orchestra.run:Running command with timeout 360 2022-03-08T18:31:57.766 DEBUG:teuthology.orchestra.run.smithi137:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2022-03-08T18:31:57.794 INFO:teuthology.orchestra.run.smithi002.stdout: 8 Mar 18:31:57 ntpd[8919]: ntpd 4.2.8p12@1.3728-o (1): Starting 2022-03-08T18:31:57.795 INFO:teuthology.orchestra.run.smithi002.stdout: 8 Mar 18:31:57 ntpd[8919]: Command line: ntpd -gq 2022-03-08T18:31:57.795 INFO:teuthology.orchestra.run.smithi002.stdout: 8 Mar 18:31:57 ntpd[8919]: proto: precision = 0.090 usec (-23) 2022-03-08T18:31:57.796 INFO:teuthology.orchestra.run.smithi002.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2022-03-08T18:31:57.796 INFO:teuthology.orchestra.run.smithi002.stdout: 8 Mar 18:31:57 ntpd[8919]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2022-03-08T18:31:57.797 INFO:teuthology.orchestra.run.smithi002.stderr:restrict ::: KOD does nothing without LIMITED. 2022-03-08T18:31:57.798 INFO:teuthology.orchestra.run.smithi002.stdout: 8 Mar 18:31:57 ntpd[8919]: restrict ::: KOD does nothing without LIMITED. 2022-03-08T18:31:57.798 INFO:teuthology.orchestra.run.smithi002.stdout: 8 Mar 18:31:57 ntpd[8919]: Listen and drop on 0 v6wildcard [::]:123 2022-03-08T18:31:57.798 INFO:teuthology.orchestra.run.smithi002.stdout: 8 Mar 18:31:57 ntpd[8919]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2022-03-08T18:31:57.799 INFO:teuthology.orchestra.run.smithi002.stdout: 8 Mar 18:31:57 ntpd[8919]: Listen normally on 2 lo 127.0.0.1:123 2022-03-08T18:31:57.799 INFO:teuthology.orchestra.run.smithi002.stdout: 8 Mar 18:31:57 ntpd[8919]: Listen normally on 3 ens1f0 172.21.15.2:123 2022-03-08T18:31:57.799 INFO:teuthology.orchestra.run.smithi002.stdout: 8 Mar 18:31:57 ntpd[8919]: Listen normally on 4 lo [::1]:123 2022-03-08T18:31:57.799 INFO:teuthology.orchestra.run.smithi002.stdout: 8 Mar 18:31:57 ntpd[8919]: Listen normally on 5 ens1f0 [fe80::ec4:7aff:febd:1920%4]:123 2022-03-08T18:31:57.800 INFO:teuthology.orchestra.run.smithi002.stdout: 8 Mar 18:31:57 ntpd[8919]: Listening on routing socket on fd #22 for interface updates 2022-03-08T18:31:57.801 INFO:teuthology.orchestra.run.smithi066.stdout: 8 Mar 18:31:57 ntpd[9065]: ntpd 4.2.8p12@1.3728-o (1): Starting 2022-03-08T18:31:57.801 INFO:teuthology.orchestra.run.smithi066.stdout: 8 Mar 18:31:57 ntpd[9065]: Command line: ntpd -gq 2022-03-08T18:31:57.801 INFO:teuthology.orchestra.run.smithi066.stdout: 8 Mar 18:31:57 ntpd[9065]: proto: precision = 0.062 usec (-24) 2022-03-08T18:31:57.802 INFO:teuthology.orchestra.run.smithi137.stdout: 8 Mar 18:31:57 ntpd[9014]: ntpd 4.2.8p12@1.3728-o (1): Starting 2022-03-08T18:31:57.802 INFO:teuthology.orchestra.run.smithi137.stdout: 8 Mar 18:31:57 ntpd[9014]: Command line: ntpd -gq 2022-03-08T18:31:57.802 INFO:teuthology.orchestra.run.smithi137.stdout: 8 Mar 18:31:57 ntpd[9014]: proto: precision = 0.083 usec (-23) 2022-03-08T18:31:57.803 INFO:teuthology.orchestra.run.smithi066.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2022-03-08T18:31:57.803 INFO:teuthology.orchestra.run.smithi137.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2022-03-08T18:31:57.804 INFO:teuthology.orchestra.run.smithi066.stdout: 8 Mar 18:31:57 ntpd[9065]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2022-03-08T18:31:57.804 INFO:teuthology.orchestra.run.smithi066.stdout: 8 Mar 18:31:57 ntpd[9065]: restrict ::: KOD does nothing without LIMITED. 2022-03-08T18:31:57.805 INFO:teuthology.orchestra.run.smithi137.stdout: 8 Mar 18:31:57 ntpd[9014]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2022-03-08T18:31:57.805 INFO:teuthology.orchestra.run.smithi137.stdout: 8 Mar 18:31:57 ntpd[9014]: restrict ::: KOD does nothing without LIMITED. 2022-03-08T18:31:57.805 INFO:teuthology.orchestra.run.smithi066.stderr:restrict ::: KOD does nothing without LIMITED. 2022-03-08T18:31:57.806 INFO:teuthology.orchestra.run.smithi137.stderr:restrict ::: KOD does nothing without LIMITED. 2022-03-08T18:31:57.807 INFO:teuthology.orchestra.run.smithi066.stdout: 8 Mar 18:31:57 ntpd[9065]: Listen and drop on 0 v6wildcard [::]:123 2022-03-08T18:31:57.808 INFO:teuthology.orchestra.run.smithi066.stdout: 8 Mar 18:31:57 ntpd[9065]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2022-03-08T18:31:57.808 INFO:teuthology.orchestra.run.smithi066.stdout: 8 Mar 18:31:57 ntpd[9065]: Listen normally on 2 lo 127.0.0.1:123 2022-03-08T18:31:57.808 INFO:teuthology.orchestra.run.smithi066.stdout: 8 Mar 18:31:57 ntpd[9065]: Listen normally on 3 enp3s0f1 172.21.15.66:123 2022-03-08T18:31:57.808 INFO:teuthology.orchestra.run.smithi066.stdout: 8 Mar 18:31:57 ntpd[9065]: Listen normally on 4 lo [::1]:123 2022-03-08T18:31:57.809 INFO:teuthology.orchestra.run.smithi066.stdout: 8 Mar 18:31:57 ntpd[9065]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:feb6:f8bd%5]:123 2022-03-08T18:31:57.809 INFO:teuthology.orchestra.run.smithi066.stdout: 8 Mar 18:31:57 ntpd[9065]: Listening on routing socket on fd #22 for interface updates 2022-03-08T18:31:57.809 INFO:teuthology.orchestra.run.smithi137.stdout: 8 Mar 18:31:57 ntpd[9014]: Listen and drop on 0 v6wildcard [::]:123 2022-03-08T18:31:57.809 INFO:teuthology.orchestra.run.smithi137.stdout: 8 Mar 18:31:57 ntpd[9014]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2022-03-08T18:31:57.810 INFO:teuthology.orchestra.run.smithi137.stdout: 8 Mar 18:31:57 ntpd[9014]: Listen normally on 2 lo 127.0.0.1:123 2022-03-08T18:31:57.810 INFO:teuthology.orchestra.run.smithi137.stdout: 8 Mar 18:31:57 ntpd[9014]: Listen normally on 3 enp3s0f1 172.21.15.137:123 2022-03-08T18:31:57.810 INFO:teuthology.orchestra.run.smithi137.stdout: 8 Mar 18:31:57 ntpd[9014]: Listen normally on 4 lo [::1]:123 2022-03-08T18:31:57.810 INFO:teuthology.orchestra.run.smithi137.stdout: 8 Mar 18:31:57 ntpd[9014]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:8137%5]:123 2022-03-08T18:31:57.811 INFO:teuthology.orchestra.run.smithi137.stdout: 8 Mar 18:31:57 ntpd[9014]: Listening on routing socket on fd #22 for interface updates 2022-03-08T18:31:58.795 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:58 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:58.795 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:58 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:58.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:58 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:58.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:58 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:58.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:58 ntpd[8919]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:31:58.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:58 ntpd[8919]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:31:58.797 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:31:58 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:58.797 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:31:58 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:58.798 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:31:58 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:58.798 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:31:58 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:58.798 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:31:58 ntpd[9014]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:31:58.798 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:31:58 ntpd[9014]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:31:58.799 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:58 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:58.800 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:58 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:58.800 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:58 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:58.800 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:58 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:58.801 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:58 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:58.802 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:58 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:58.802 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:58 ntpd[9065]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:31:58.802 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:58 ntpd[9065]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:31:58.802 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:58 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:58.803 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:58 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:58.803 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:58 ntpd[9065]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:31:58.804 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:58 ntpd[9065]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:31:59.795 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:59 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:59.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:59 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:59.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:59 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:59.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:59 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:59.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:59 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:59.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:59 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:59.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:59 ntpd[8919]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:31:59.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:59 ntpd[8919]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:31:59.798 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:59 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:59.798 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:59 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:59.798 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:59 ntpd[8919]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:31:59.798 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:31:59 ntpd[8919]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:31:59.800 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:59 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:59.800 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:59 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:59.800 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:59 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:31:59.800 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:59 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:31:59.801 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:59 ntpd[9065]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:31:59.801 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:31:59 ntpd[9065]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:00.795 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:00 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:00.795 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:00 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:00.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:00 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:00.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:00 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:00.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:00 ntpd[8919]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:00.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:00 ntpd[8919]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:00.798 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:00 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:00.798 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:00 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:00.799 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:00 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:00.799 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:00 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:00.799 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:00 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:00.800 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:00 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:00.800 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:00 ntpd[9014]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:00.800 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:00 ntpd[9014]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:00.801 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:00 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:00.801 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:00 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:00.801 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:00 ntpd[9014]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:00.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:00 ntpd[9014]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:00.803 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:00 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:00.804 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:00 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:00.804 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:00 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:00.804 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:00 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:00.804 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:00 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:00.804 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:00 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:00.805 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:00 ntpd[9065]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:00.805 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:00 ntpd[9065]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:00.805 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:00 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:00.805 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:00 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:00.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:00 ntpd[9065]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:00.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:00 ntpd[9065]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:01.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:01.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:01.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:01.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:01.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:01.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:01.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:01.798 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:01.798 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:01.798 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:01.798 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:01.799 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:01.799 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:01.799 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:01.799 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:01.800 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:01.800 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:01.800 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:01 ntpd[8919]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:01.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:01 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:01.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:01 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:01.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:01 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:01.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:01 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:01.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:01 ntpd[9014]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:01.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:01 ntpd[9014]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:01.805 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:01 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:01.805 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:01 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:01.805 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:01 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:01.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:01 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:01.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:01 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:01.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:01 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:01.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:01 ntpd[9065]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:01.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:01 ntpd[9065]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:01.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:01 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:01.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:01 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:01.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:01 ntpd[9065]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:01.808 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:01 ntpd[9065]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:02.795 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:02 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:02.795 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:02 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:02.795 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:02 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:02.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:02 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:02.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:02 ntpd[8919]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:02.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:02 ntpd[8919]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:02.799 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:02.800 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:02.800 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:02.800 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:02.801 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:02.801 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:02.801 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:02.801 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:02.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:02.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:02.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:02.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:02.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:02.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:02.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:02.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:02.804 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:02.804 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:02 ntpd[9014]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:02.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:02 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:02.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:02 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:02.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:02 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:02.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:02 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:02.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:02 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:02.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:02 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:02.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:02 ntpd[9065]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:02.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:02 ntpd[9065]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:02.808 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:02 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:02.808 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:02 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:02.808 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:02 ntpd[9065]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:02.808 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:02 ntpd[9065]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:03.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:03.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:03.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:03.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:03.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:03.798 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:03.798 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:03.798 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:03.798 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:03.799 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:03.799 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:03.799 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:03.799 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:03.800 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:03.800 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:03.800 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:03.800 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:03.800 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:03 ntpd[8919]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:03.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:03 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:03.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:03 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:03.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:03 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:03.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:03 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:03.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:03 ntpd[9014]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:03.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:03 ntpd[9014]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:03.804 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:03 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:03.805 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:03 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:03.805 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:03 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:03.805 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:03 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:03.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:03 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:03.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:03 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:03.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:03 ntpd[9065]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:03.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:03 ntpd[9065]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:03.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:03 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:03.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:03 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:03.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:03 ntpd[9065]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:03.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:03 ntpd[9065]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:04.795 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:04 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:04.795 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:04 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:04.795 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:04 ntpd[8919]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:04.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:04 ntpd[8919]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:04.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:04 ntpd[8919]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:04.796 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:04 ntpd[8919]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:04.797 INFO:teuthology.orchestra.run.smithi002.stdout: 8 Mar 18:32:04 ntpd[8919]: ntpd: time slew -0.006657 s 2022-03-08T18:32:04.797 INFO:teuthology.orchestra.run.smithi002.stdout:ntpd: time slew -0.006657s 2022-03-08T18:32:04.797 INFO:teuthology.orchestra.run.smithi002.stderr: 8 Mar 18:32:04 ntpd[8919]: can't open /var/log/ntpstats/loopstats.20220308: Permission denied 2022-03-08T18:32:04.799 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:04.799 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:04.799 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:04.801 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:04.801 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:04.801 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:04.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:04.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:04.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:04.802 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:04.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:04.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:04.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:04.803 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:04.804 INFO:teuthology.orchestra.run.smithi137.stdout: 8 Mar 18:32:04 ntpd[9014]: ntpd: time slew -0.002500 s 2022-03-08T18:32:04.804 INFO:teuthology.orchestra.run.smithi137.stdout:ntpd: time slew -0.002500s 2022-03-08T18:32:04.804 INFO:teuthology.orchestra.run.smithi137.stderr: 8 Mar 18:32:04 ntpd[9014]: can't open /var/log/ntpstats/loopstats.20220308: Permission denied 2022-03-08T18:32:04.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:04 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:04.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:04 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:04.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:04 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:04.806 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:04 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:04.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:04 ntpd[9065]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-03-08T18:32:04.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:04 ntpd[9065]: can't open /var/log/ntpstats/rawstats.20220308: Permission denied 2022-03-08T18:32:04.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:04 ntpd[9065]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-03-08T18:32:04.807 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:04 ntpd[9065]: can't open /var/log/ntpstats/peerstats.20220308: Permission denied 2022-03-08T18:32:04.808 INFO:teuthology.orchestra.run.smithi066.stdout: 8 Mar 18:32:04 ntpd[9065]: ntpd: time slew -0.005708 s 2022-03-08T18:32:04.808 INFO:teuthology.orchestra.run.smithi066.stdout:ntpd: time slew -0.005708s 2022-03-08T18:32:04.809 INFO:teuthology.orchestra.run.smithi066.stderr: 8 Mar 18:32:04 ntpd[9065]: can't open /var/log/ntpstats/loopstats.20220308: Permission denied 2022-03-08T18:32:04.856 INFO:teuthology.orchestra.run.smithi002.stdout: remote refid st t when poll reach delay offset jitter 2022-03-08T18:32:04.856 INFO:teuthology.orchestra.run.smithi002.stdout:============================================================================== 2022-03-08T18:32:04.857 INFO:teuthology.orchestra.run.smithi002.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-03-08T18:32:04.857 INFO:teuthology.orchestra.run.smithi002.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-03-08T18:32:04.857 INFO:teuthology.orchestra.run.smithi002.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-03-08T18:32:04.857 INFO:teuthology.orchestra.run.smithi002.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-03-08T18:32:04.862 INFO:teuthology.orchestra.run.smithi066.stdout: remote refid st t when poll reach delay offset jitter 2022-03-08T18:32:04.862 INFO:teuthology.orchestra.run.smithi066.stdout:============================================================================== 2022-03-08T18:32:04.862 INFO:teuthology.orchestra.run.smithi066.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-03-08T18:32:04.862 INFO:teuthology.orchestra.run.smithi066.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-03-08T18:32:04.863 INFO:teuthology.orchestra.run.smithi066.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-03-08T18:32:04.863 INFO:teuthology.orchestra.run.smithi066.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-03-08T18:32:04.940 INFO:teuthology.orchestra.run.smithi137.stdout: remote refid st t when poll reach delay offset jitter 2022-03-08T18:32:04.940 INFO:teuthology.orchestra.run.smithi137.stdout:============================================================================== 2022-03-08T18:32:04.940 INFO:teuthology.orchestra.run.smithi137.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-03-08T18:32:04.940 INFO:teuthology.orchestra.run.smithi137.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-03-08T18:32:04.941 INFO:teuthology.orchestra.run.smithi137.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-03-08T18:32:04.941 INFO:teuthology.orchestra.run.smithi137.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-03-08T18:32:04.941 INFO:teuthology.run_tasks:Running task kubeadm... 2022-03-08T18:32:04.956 INFO:tasks.kubeadm:Kubeadm start 2022-03-08T18:32:04.956 INFO:tasks.kubeadm:Config: {'pod_network': 'calico', 'version': '1.21'} 2022-03-08T18:32:04.957 DEBUG:teuthology.orchestra.run.smithi002:> sudo modprobe br_netfilter && sudo sysctl net.bridge.bridge-nf-call-ip6tables=1 && sudo sysctl net.bridge.bridge-nf-call-iptables=1 && sudo sysctl net.ipv4.ip_forward=1 && sudo swapoff -a 2022-03-08T18:32:04.960 DEBUG:teuthology.orchestra.run.smithi066:> sudo modprobe br_netfilter && sudo sysctl net.bridge.bridge-nf-call-ip6tables=1 && sudo sysctl net.bridge.bridge-nf-call-iptables=1 && sudo sysctl net.ipv4.ip_forward=1 && sudo swapoff -a 2022-03-08T18:32:04.962 DEBUG:teuthology.orchestra.run.smithi137:> sudo modprobe br_netfilter && sudo sysctl net.bridge.bridge-nf-call-ip6tables=1 && sudo sysctl net.bridge.bridge-nf-call-iptables=1 && sudo sysctl net.ipv4.ip_forward=1 && sudo swapoff -a 2022-03-08T18:32:04.985 INFO:teuthology.orchestra.run.smithi002.stdout:net.bridge.bridge-nf-call-ip6tables = 1 2022-03-08T18:32:04.987 INFO:teuthology.orchestra.run.smithi066.stdout:net.bridge.bridge-nf-call-ip6tables = 1 2022-03-08T18:32:04.994 INFO:teuthology.orchestra.run.smithi002.stdout:net.bridge.bridge-nf-call-iptables = 1 2022-03-08T18:32:04.997 INFO:teuthology.orchestra.run.smithi066.stdout:net.bridge.bridge-nf-call-iptables = 1 2022-03-08T18:32:05.003 INFO:teuthology.orchestra.run.smithi002.stdout:net.ipv4.ip_forward = 1 2022-03-08T18:32:05.005 INFO:teuthology.orchestra.run.smithi066.stdout:net.ipv4.ip_forward = 1 2022-03-08T18:32:05.012 INFO:teuthology.orchestra.run.smithi137.stdout:net.bridge.bridge-nf-call-ip6tables = 1 2022-03-08T18:32:05.019 INFO:teuthology.orchestra.run.smithi137.stdout:net.bridge.bridge-nf-call-iptables = 1 2022-03-08T18:32:05.027 INFO:teuthology.orchestra.run.smithi137.stdout:net.ipv4.ip_forward = 1 2022-03-08T18:32:05.084 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:32:05.085 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd if=/etc/docker/daemon.json of=/dev/stdout 2022-03-08T18:32:05.101 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:32:05.101 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd of=/etc/docker/daemon.json 2022-03-08T18:32:05.158 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:32:05.159 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/etc/docker/daemon.json of=/dev/stdout 2022-03-08T18:32:05.175 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:32:05.176 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/docker/daemon.json 2022-03-08T18:32:05.235 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:32:05.236 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd if=/etc/docker/daemon.json of=/dev/stdout 2022-03-08T18:32:05.253 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:32:05.254 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd of=/etc/docker/daemon.json 2022-03-08T18:32:05.270 DEBUG:teuthology.orchestra.run.smithi002:> sudo systemctl restart docker || true 2022-03-08T18:32:05.273 DEBUG:teuthology.orchestra.run.smithi066:> sudo systemctl restart docker || true 2022-03-08T18:32:05.275 DEBUG:teuthology.orchestra.run.smithi137:> sudo systemctl restart docker || true 2022-03-08T18:32:06.494 INFO:tasks.kubeadm:Identifying pod subnet 2022-03-08T18:32:06.501 INFO:tasks.kubeadm:172.21.15.2 is in 172.21.0.0/20 at pos 3841 2022-03-08T18:32:06.520 INFO:tasks.kubeadm:Pod subnet: 10.248.8.0/21 2022-03-08T18:32:06.520 INFO:tasks.kubeadm:Installing kube{adm,ctl,let} 2022-03-08T18:32:06.521 DEBUG:teuthology.orchestra.run.smithi002:> sudo apt update && sudo apt install -y apt-transport-https ca-certificates curl && sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && echo 'deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main' | sudo tee /etc/apt/sources.list.d/kubernetes.list && sudo apt update && sudo apt install -y kubelet kubeadm kubectl bridge-utils 2022-03-08T18:32:06.524 DEBUG:teuthology.orchestra.run.smithi066:> sudo apt update && sudo apt install -y apt-transport-https ca-certificates curl && sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && echo 'deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main' | sudo tee /etc/apt/sources.list.d/kubernetes.list && sudo apt update && sudo apt install -y kubelet kubeadm kubectl bridge-utils 2022-03-08T18:32:06.526 DEBUG:teuthology.orchestra.run.smithi137:> sudo apt update && sudo apt install -y apt-transport-https ca-certificates curl && sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && echo 'deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main' | sudo tee /etc/apt/sources.list.d/kubernetes.list && sudo apt update && sudo apt install -y kubelet kubeadm kubectl bridge-utils 2022-03-08T18:32:06.549 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:32:06.550 INFO:teuthology.orchestra.run.smithi002.stderr:WARNING: apt does not have a stable CLI interface. Use with caution in scripts. 2022-03-08T18:32:06.550 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:32:06.551 INFO:teuthology.orchestra.run.smithi066.stderr: 2022-03-08T18:32:06.551 INFO:teuthology.orchestra.run.smithi066.stderr:WARNING: apt does not have a stable CLI interface. Use with caution in scripts. 2022-03-08T18:32:06.552 INFO:teuthology.orchestra.run.smithi066.stderr: 2022-03-08T18:32:06.555 INFO:teuthology.orchestra.run.smithi137.stderr: 2022-03-08T18:32:06.555 INFO:teuthology.orchestra.run.smithi137.stderr:WARNING: apt does not have a stable CLI interface. Use with caution in scripts. 2022-03-08T18:32:06.555 INFO:teuthology.orchestra.run.smithi137.stderr: 2022-03-08T18:32:06.664 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2022-03-08T18:32:06.779 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:32:06.796 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:32:06.797 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:2 http://security.ubuntu.com/ubuntu focal-security InRelease 2022-03-08T18:32:06.797 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2022-03-08T18:32:06.798 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:32:06.861 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2022-03-08T18:32:06.868 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2022-03-08T18:32:06.869 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2022-03-08T18:32:06.943 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2022-03-08T18:32:06.948 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2022-03-08T18:32:06.950 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2022-03-08T18:32:08.183 INFO:teuthology.orchestra.run.smithi002.stdout:Reading package lists... 2022-03-08T18:32:08.275 INFO:teuthology.orchestra.run.smithi137.stdout:Reading package lists... 2022-03-08T18:32:08.309 INFO:teuthology.orchestra.run.smithi066.stdout:Reading package lists... 2022-03-08T18:32:08.357 INFO:teuthology.orchestra.run.smithi002.stdout:Building dependency tree... 2022-03-08T18:32:08.359 INFO:teuthology.orchestra.run.smithi002.stdout:Reading state information... 2022-03-08T18:32:08.449 INFO:teuthology.orchestra.run.smithi002.stdout:85 packages can be upgraded. Run 'apt list --upgradable' to see them. 2022-03-08T18:32:08.450 INFO:teuthology.orchestra.run.smithi137.stdout:Building dependency tree... 2022-03-08T18:32:08.450 INFO:teuthology.orchestra.run.smithi137.stdout:Reading state information... 2022-03-08T18:32:08.465 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:32:08.466 INFO:teuthology.orchestra.run.smithi002.stderr:WARNING: apt does not have a stable CLI interface. Use with caution in scripts. 2022-03-08T18:32:08.466 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:32:08.487 INFO:teuthology.orchestra.run.smithi066.stdout:Building dependency tree... 2022-03-08T18:32:08.488 INFO:teuthology.orchestra.run.smithi066.stdout:Reading state information... 2022-03-08T18:32:08.522 INFO:teuthology.orchestra.run.smithi002.stdout:Reading package lists... 2022-03-08T18:32:08.537 INFO:teuthology.orchestra.run.smithi137.stdout:85 packages can be upgraded. Run 'apt list --upgradable' to see them. 2022-03-08T18:32:08.550 INFO:teuthology.orchestra.run.smithi137.stderr: 2022-03-08T18:32:08.550 INFO:teuthology.orchestra.run.smithi137.stderr:WARNING: apt does not have a stable CLI interface. Use with caution in scripts. 2022-03-08T18:32:08.550 INFO:teuthology.orchestra.run.smithi137.stderr: 2022-03-08T18:32:08.578 INFO:teuthology.orchestra.run.smithi066.stdout:85 packages can be upgraded. Run 'apt list --upgradable' to see them. 2022-03-08T18:32:08.592 INFO:teuthology.orchestra.run.smithi066.stderr: 2022-03-08T18:32:08.593 INFO:teuthology.orchestra.run.smithi066.stderr:WARNING: apt does not have a stable CLI interface. Use with caution in scripts. 2022-03-08T18:32:08.594 INFO:teuthology.orchestra.run.smithi066.stderr: 2022-03-08T18:32:08.602 INFO:teuthology.orchestra.run.smithi137.stdout:Reading package lists... 2022-03-08T18:32:08.644 INFO:teuthology.orchestra.run.smithi066.stdout:Reading package lists... 2022-03-08T18:32:08.699 INFO:teuthology.orchestra.run.smithi002.stdout:Building dependency tree... 2022-03-08T18:32:08.700 INFO:teuthology.orchestra.run.smithi002.stdout:Reading state information... 2022-03-08T18:32:08.761 INFO:teuthology.orchestra.run.smithi137.stdout:Building dependency tree... 2022-03-08T18:32:08.762 INFO:teuthology.orchestra.run.smithi137.stdout:Reading state information... 2022-03-08T18:32:08.807 INFO:teuthology.orchestra.run.smithi066.stdout:Building dependency tree... 2022-03-08T18:32:08.808 INFO:teuthology.orchestra.run.smithi066.stdout:Reading state information... 2022-03-08T18:32:08.889 INFO:teuthology.orchestra.run.smithi002.stdout:ca-certificates is already the newest version (20210119~20.04.2). 2022-03-08T18:32:08.890 INFO:teuthology.orchestra.run.smithi002.stdout:curl is already the newest version (7.68.0-1ubuntu2.7). 2022-03-08T18:32:08.890 INFO:teuthology.orchestra.run.smithi002.stdout:The following NEW packages will be installed: 2022-03-08T18:32:08.891 INFO:teuthology.orchestra.run.smithi002.stdout: apt-transport-https 2022-03-08T18:32:08.924 INFO:teuthology.orchestra.run.smithi137.stdout:ca-certificates is already the newest version (20210119~20.04.2). 2022-03-08T18:32:08.924 INFO:teuthology.orchestra.run.smithi137.stdout:curl is already the newest version (7.68.0-1ubuntu2.7). 2022-03-08T18:32:08.925 INFO:teuthology.orchestra.run.smithi137.stdout:The following NEW packages will be installed: 2022-03-08T18:32:08.926 INFO:teuthology.orchestra.run.smithi137.stdout: apt-transport-https 2022-03-08T18:32:08.975 INFO:teuthology.orchestra.run.smithi066.stdout:ca-certificates is already the newest version (20210119~20.04.2). 2022-03-08T18:32:08.975 INFO:teuthology.orchestra.run.smithi066.stdout:curl is already the newest version (7.68.0-1ubuntu2.7). 2022-03-08T18:32:08.976 INFO:teuthology.orchestra.run.smithi066.stdout:The following NEW packages will be installed: 2022-03-08T18:32:08.977 INFO:teuthology.orchestra.run.smithi066.stdout: apt-transport-https 2022-03-08T18:32:09.077 INFO:teuthology.orchestra.run.smithi002.stdout:0 upgraded, 1 newly installed, 0 to remove and 85 not upgraded. 2022-03-08T18:32:09.078 INFO:teuthology.orchestra.run.smithi002.stdout:Need to get 4,680 B of archives. 2022-03-08T18:32:09.078 INFO:teuthology.orchestra.run.smithi002.stdout:After this operation, 162 kB of additional disk space will be used. 2022-03-08T18:32:09.079 INFO:teuthology.orchestra.run.smithi002.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 apt-transport-https all 2.0.6 [4,680 B] 2022-03-08T18:32:09.111 INFO:teuthology.orchestra.run.smithi137.stdout:0 upgraded, 1 newly installed, 0 to remove and 85 not upgraded. 2022-03-08T18:32:09.111 INFO:teuthology.orchestra.run.smithi137.stdout:Need to get 4,680 B of archives. 2022-03-08T18:32:09.112 INFO:teuthology.orchestra.run.smithi137.stdout:After this operation, 162 kB of additional disk space will be used. 2022-03-08T18:32:09.112 INFO:teuthology.orchestra.run.smithi137.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 apt-transport-https all 2.0.6 [4,680 B] 2022-03-08T18:32:09.158 INFO:teuthology.orchestra.run.smithi066.stdout:0 upgraded, 1 newly installed, 0 to remove and 85 not upgraded. 2022-03-08T18:32:09.159 INFO:teuthology.orchestra.run.smithi066.stdout:Need to get 4,680 B of archives. 2022-03-08T18:32:09.159 INFO:teuthology.orchestra.run.smithi066.stdout:After this operation, 162 kB of additional disk space will be used. 2022-03-08T18:32:09.159 INFO:teuthology.orchestra.run.smithi066.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 apt-transport-https all 2.0.6 [4,680 B] 2022-03-08T18:32:09.316 INFO:teuthology.orchestra.run.smithi002.stderr:debconf: unable to initialize frontend: Dialog 2022-03-08T18:32:09.317 INFO:teuthology.orchestra.run.smithi002.stderr:debconf: (Dialog frontend will not work on a dumb terminal, an emacs shell buffer, or without a controlling terminal.) 2022-03-08T18:32:09.317 INFO:teuthology.orchestra.run.smithi002.stderr:debconf: falling back to frontend: Readline 2022-03-08T18:32:09.335 INFO:teuthology.orchestra.run.smithi002.stderr:debconf: unable to initialize frontend: Readline 2022-03-08T18:32:09.335 INFO:teuthology.orchestra.run.smithi002.stderr:debconf: (This frontend requires a controlling tty.) 2022-03-08T18:32:09.336 INFO:teuthology.orchestra.run.smithi002.stderr:debconf: falling back to frontend: Teletype 2022-03-08T18:32:09.337 INFO:teuthology.orchestra.run.smithi137.stderr:debconf: unable to initialize frontend: Dialog 2022-03-08T18:32:09.337 INFO:teuthology.orchestra.run.smithi137.stderr:debconf: (Dialog frontend will not work on a dumb terminal, an emacs shell buffer, or without a controlling terminal.) 2022-03-08T18:32:09.338 INFO:teuthology.orchestra.run.smithi137.stderr:debconf: falling back to frontend: Readline 2022-03-08T18:32:09.341 INFO:teuthology.orchestra.run.smithi002.stderr:dpkg-preconfigure: unable to re-open stdin: 2022-03-08T18:32:09.355 INFO:teuthology.orchestra.run.smithi137.stderr:debconf: unable to initialize frontend: Readline 2022-03-08T18:32:09.356 INFO:teuthology.orchestra.run.smithi137.stderr:debconf: (This frontend requires a controlling tty.) 2022-03-08T18:32:09.356 INFO:teuthology.orchestra.run.smithi137.stderr:debconf: falling back to frontend: Teletype 2022-03-08T18:32:09.362 INFO:teuthology.orchestra.run.smithi137.stderr:dpkg-preconfigure: unable to re-open stdin: 2022-03-08T18:32:09.380 INFO:teuthology.orchestra.run.smithi066.stderr:debconf: unable to initialize frontend: Dialog 2022-03-08T18:32:09.380 INFO:teuthology.orchestra.run.smithi066.stderr:debconf: (Dialog frontend will not work on a dumb terminal, an emacs shell buffer, or without a controlling terminal.) 2022-03-08T18:32:09.381 INFO:teuthology.orchestra.run.smithi066.stderr:debconf: falling back to frontend: Readline 2022-03-08T18:32:09.391 INFO:teuthology.orchestra.run.smithi002.stdout:Fetched 4,680 B in 0s (26.2 kB/s) 2022-03-08T18:32:09.400 INFO:teuthology.orchestra.run.smithi066.stderr:debconf: unable to initialize frontend: Readline 2022-03-08T18:32:09.401 INFO:teuthology.orchestra.run.smithi066.stderr:debconf: (This frontend requires a controlling tty.) 2022-03-08T18:32:09.401 INFO:teuthology.orchestra.run.smithi066.stderr:debconf: falling back to frontend: Teletype 2022-03-08T18:32:09.407 INFO:teuthology.orchestra.run.smithi066.stderr:dpkg-preconfigure: unable to re-open stdin: 2022-03-08T18:32:09.410 INFO:teuthology.orchestra.run.smithi137.stdout:Fetched 4,680 B in 0s (26.5 kB/s) 2022-03-08T18:32:09.460 INFO:teuthology.orchestra.run.smithi066.stdout:Fetched 4,680 B in 0s (26.7 kB/s) 2022-03-08T18:32:09.461 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package apt-transport-https. 2022-03-08T18:32:09.499 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package apt-transport-https. 2022-03-08T18:32:09.504 INFO:teuthology.orchestra.run.smithi002.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 109240 files and directories currently installed.) 2022-03-08T18:32:09.507 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../apt-transport-https_2.0.6_all.deb ... 2022-03-08T18:32:09.529 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package apt-transport-https. 2022-03-08T18:32:09.544 INFO:teuthology.orchestra.run.smithi137.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 109240 files and directories currently installed.) 2022-03-08T18:32:09.547 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../apt-transport-https_2.0.6_all.deb ... 2022-03-08T18:32:09.551 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking apt-transport-https (2.0.6) ... 2022-03-08T18:32:09.571 INFO:teuthology.orchestra.run.smithi066.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 109240 files and directories currently installed.) 2022-03-08T18:32:09.574 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../apt-transport-https_2.0.6_all.deb ... 2022-03-08T18:32:09.590 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking apt-transport-https (2.0.6) ... 2022-03-08T18:32:09.628 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking apt-transport-https (2.0.6) ... 2022-03-08T18:32:09.821 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up apt-transport-https (2.0.6) ... 2022-03-08T18:32:09.854 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up apt-transport-https (2.0.6) ... 2022-03-08T18:32:09.908 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up apt-transport-https (2.0.6) ... 2022-03-08T18:32:10.368 INFO:teuthology.orchestra.run.smithi002.stdout:deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main 2022-03-08T18:32:10.384 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:32:10.384 INFO:teuthology.orchestra.run.smithi002.stderr:WARNING: apt does not have a stable CLI interface. Use with caution in scripts. 2022-03-08T18:32:10.384 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:32:10.395 INFO:teuthology.orchestra.run.smithi137.stdout:deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main 2022-03-08T18:32:10.408 INFO:teuthology.orchestra.run.smithi137.stderr: 2022-03-08T18:32:10.408 INFO:teuthology.orchestra.run.smithi137.stderr:WARNING: apt does not have a stable CLI interface. Use with caution in scripts. 2022-03-08T18:32:10.408 INFO:teuthology.orchestra.run.smithi137.stderr: 2022-03-08T18:32:10.532 INFO:teuthology.orchestra.run.smithi066.stdout:deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main 2022-03-08T18:32:10.535 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2022-03-08T18:32:10.546 INFO:teuthology.orchestra.run.smithi066.stderr: 2022-03-08T18:32:10.547 INFO:teuthology.orchestra.run.smithi066.stderr:WARNING: apt does not have a stable CLI interface. Use with caution in scripts. 2022-03-08T18:32:10.547 INFO:teuthology.orchestra.run.smithi066.stderr: 2022-03-08T18:32:10.637 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:32:10.637 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:3 http://security.ubuntu.com/ubuntu focal-security InRelease 2022-03-08T18:32:10.649 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:32:10.675 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2022-03-08T18:32:10.707 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2022-03-08T18:32:10.732 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2022-03-08T18:32:10.788 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:32:10.789 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:5 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2022-03-08T18:32:10.821 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:5 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2022-03-08T18:32:10.874 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2022-03-08T18:32:10.956 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:5 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2022-03-08T18:32:11.401 INFO:teuthology.orchestra.run.smithi137.stdout:Get:2 https://packages.cloud.google.com/apt kubernetes-xenial InRelease [9,383 B] 2022-03-08T18:32:11.496 INFO:teuthology.orchestra.run.smithi137.stdout:Get:6 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 Packages [53.9 kB] 2022-03-08T18:32:11.534 INFO:teuthology.orchestra.run.smithi066.stdout:Get:2 https://packages.cloud.google.com/apt kubernetes-xenial InRelease [9,383 B] 2022-03-08T18:32:11.535 INFO:teuthology.orchestra.run.smithi002.stdout:Get:1 https://packages.cloud.google.com/apt kubernetes-xenial InRelease [9,383 B] 2022-03-08T18:32:11.633 INFO:teuthology.orchestra.run.smithi137.stdout:Fetched 63.3 kB in 1s (58.7 kB/s) 2022-03-08T18:32:11.634 INFO:teuthology.orchestra.run.smithi066.stdout:Get:6 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 Packages [53.9 kB] 2022-03-08T18:32:11.648 INFO:teuthology.orchestra.run.smithi002.stdout:Get:6 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 Packages [53.9 kB] 2022-03-08T18:32:11.778 INFO:teuthology.orchestra.run.smithi066.stdout:Fetched 63.3 kB in 1s (58.5 kB/s) 2022-03-08T18:32:11.786 INFO:teuthology.orchestra.run.smithi002.stdout:Fetched 63.3 kB in 1s (50.5 kB/s) 2022-03-08T18:32:12.480 INFO:teuthology.orchestra.run.smithi137.stdout:Reading package lists... 2022-03-08T18:32:12.650 INFO:teuthology.orchestra.run.smithi066.stdout:Reading package lists... 2022-03-08T18:32:12.654 INFO:teuthology.orchestra.run.smithi137.stdout:Building dependency tree... 2022-03-08T18:32:12.654 INFO:teuthology.orchestra.run.smithi137.stdout:Reading state information... 2022-03-08T18:32:12.664 INFO:teuthology.orchestra.run.smithi002.stdout:Reading package lists... 2022-03-08T18:32:12.743 INFO:teuthology.orchestra.run.smithi137.stdout:85 packages can be upgraded. Run 'apt list --upgradable' to see them. 2022-03-08T18:32:12.759 INFO:teuthology.orchestra.run.smithi137.stderr: 2022-03-08T18:32:12.760 INFO:teuthology.orchestra.run.smithi137.stderr:WARNING: apt does not have a stable CLI interface. Use with caution in scripts. 2022-03-08T18:32:12.760 INFO:teuthology.orchestra.run.smithi137.stderr: 2022-03-08T18:32:12.816 INFO:teuthology.orchestra.run.smithi137.stdout:Reading package lists... 2022-03-08T18:32:12.826 INFO:teuthology.orchestra.run.smithi066.stdout:Building dependency tree... 2022-03-08T18:32:12.827 INFO:teuthology.orchestra.run.smithi066.stdout:Reading state information... 2022-03-08T18:32:12.845 INFO:teuthology.orchestra.run.smithi002.stdout:Building dependency tree... 2022-03-08T18:32:12.845 INFO:teuthology.orchestra.run.smithi002.stdout:Reading state information... 2022-03-08T18:32:12.918 INFO:teuthology.orchestra.run.smithi066.stdout:85 packages can be upgraded. Run 'apt list --upgradable' to see them. 2022-03-08T18:32:12.934 INFO:teuthology.orchestra.run.smithi066.stderr: 2022-03-08T18:32:12.934 INFO:teuthology.orchestra.run.smithi066.stderr:WARNING: apt does not have a stable CLI interface. Use with caution in scripts. 2022-03-08T18:32:12.934 INFO:teuthology.orchestra.run.smithi066.stderr: 2022-03-08T18:32:12.940 INFO:teuthology.orchestra.run.smithi002.stdout:85 packages can be upgraded. Run 'apt list --upgradable' to see them. 2022-03-08T18:32:12.955 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:32:12.955 INFO:teuthology.orchestra.run.smithi002.stderr:WARNING: apt does not have a stable CLI interface. Use with caution in scripts. 2022-03-08T18:32:12.955 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:32:12.988 INFO:teuthology.orchestra.run.smithi066.stdout:Reading package lists... 2022-03-08T18:32:12.992 INFO:teuthology.orchestra.run.smithi137.stdout:Building dependency tree... 2022-03-08T18:32:12.992 INFO:teuthology.orchestra.run.smithi137.stdout:Reading state information... 2022-03-08T18:32:13.013 INFO:teuthology.orchestra.run.smithi002.stdout:Reading package lists... 2022-03-08T18:32:13.143 INFO:teuthology.orchestra.run.smithi137.stdout:bridge-utils is already the newest version (1.6-2ubuntu1). 2022-03-08T18:32:13.143 INFO:teuthology.orchestra.run.smithi137.stdout:bridge-utils set to manually installed. 2022-03-08T18:32:13.143 INFO:teuthology.orchestra.run.smithi137.stdout:The following additional packages will be installed: 2022-03-08T18:32:13.144 INFO:teuthology.orchestra.run.smithi137.stdout: conntrack cri-tools ebtables kubernetes-cni socat 2022-03-08T18:32:13.144 INFO:teuthology.orchestra.run.smithi137.stdout:Suggested packages: 2022-03-08T18:32:13.145 INFO:teuthology.orchestra.run.smithi137.stdout: nftables 2022-03-08T18:32:13.167 INFO:teuthology.orchestra.run.smithi066.stdout:Building dependency tree... 2022-03-08T18:32:13.167 INFO:teuthology.orchestra.run.smithi066.stdout:Reading state information... 2022-03-08T18:32:13.190 INFO:teuthology.orchestra.run.smithi137.stdout:The following NEW packages will be installed: 2022-03-08T18:32:13.191 INFO:teuthology.orchestra.run.smithi137.stdout: conntrack cri-tools ebtables kubeadm kubectl kubelet kubernetes-cni socat 2022-03-08T18:32:13.191 INFO:teuthology.orchestra.run.smithi002.stdout:Building dependency tree... 2022-03-08T18:32:13.192 INFO:teuthology.orchestra.run.smithi002.stdout:Reading state information... 2022-03-08T18:32:13.324 INFO:teuthology.orchestra.run.smithi066.stdout:bridge-utils is already the newest version (1.6-2ubuntu1). 2022-03-08T18:32:13.324 INFO:teuthology.orchestra.run.smithi066.stdout:bridge-utils set to manually installed. 2022-03-08T18:32:13.324 INFO:teuthology.orchestra.run.smithi066.stdout:The following additional packages will be installed: 2022-03-08T18:32:13.325 INFO:teuthology.orchestra.run.smithi066.stdout: conntrack cri-tools ebtables kubernetes-cni socat 2022-03-08T18:32:13.326 INFO:teuthology.orchestra.run.smithi066.stdout:Suggested packages: 2022-03-08T18:32:13.326 INFO:teuthology.orchestra.run.smithi066.stdout: nftables 2022-03-08T18:32:13.343 INFO:teuthology.orchestra.run.smithi002.stdout:bridge-utils is already the newest version (1.6-2ubuntu1). 2022-03-08T18:32:13.343 INFO:teuthology.orchestra.run.smithi002.stdout:bridge-utils set to manually installed. 2022-03-08T18:32:13.343 INFO:teuthology.orchestra.run.smithi002.stdout:The following additional packages will be installed: 2022-03-08T18:32:13.344 INFO:teuthology.orchestra.run.smithi002.stdout: conntrack cri-tools ebtables kubernetes-cni socat 2022-03-08T18:32:13.344 INFO:teuthology.orchestra.run.smithi002.stdout:Suggested packages: 2022-03-08T18:32:13.344 INFO:teuthology.orchestra.run.smithi002.stdout: nftables 2022-03-08T18:32:13.375 INFO:teuthology.orchestra.run.smithi066.stdout:The following NEW packages will be installed: 2022-03-08T18:32:13.376 INFO:teuthology.orchestra.run.smithi066.stdout: conntrack cri-tools ebtables kubeadm kubectl kubelet kubernetes-cni socat 2022-03-08T18:32:13.383 INFO:teuthology.orchestra.run.smithi137.stdout:0 upgraded, 8 newly installed, 0 to remove and 85 not upgraded. 2022-03-08T18:32:13.384 INFO:teuthology.orchestra.run.smithi137.stdout:Need to get 73.6 MB of archives. 2022-03-08T18:32:13.384 INFO:teuthology.orchestra.run.smithi137.stdout:After this operation, 321 MB of additional disk space will be used. 2022-03-08T18:32:13.384 INFO:teuthology.orchestra.run.smithi137.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 conntrack amd64 1:1.4.5-2 [30.3 kB] 2022-03-08T18:32:13.394 INFO:teuthology.orchestra.run.smithi002.stdout:The following NEW packages will be installed: 2022-03-08T18:32:13.395 INFO:teuthology.orchestra.run.smithi002.stdout: conntrack cri-tools ebtables kubeadm kubectl kubelet kubernetes-cni socat 2022-03-08T18:32:13.438 INFO:teuthology.orchestra.run.smithi137.stdout:Get:1 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 cri-tools amd64 1.19.0-00 [11.2 MB] 2022-03-08T18:32:13.571 INFO:teuthology.orchestra.run.smithi137.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 ebtables amd64 2.0.11-3build1 [80.3 kB] 2022-03-08T18:32:13.580 INFO:teuthology.orchestra.run.smithi066.stdout:0 upgraded, 8 newly installed, 0 to remove and 85 not upgraded. 2022-03-08T18:32:13.580 INFO:teuthology.orchestra.run.smithi066.stdout:Need to get 73.6 MB of archives. 2022-03-08T18:32:13.580 INFO:teuthology.orchestra.run.smithi066.stdout:After this operation, 321 MB of additional disk space will be used. 2022-03-08T18:32:13.580 INFO:teuthology.orchestra.run.smithi066.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 conntrack amd64 1:1.4.5-2 [30.3 kB] 2022-03-08T18:32:13.606 INFO:teuthology.orchestra.run.smithi002.stdout:0 upgraded, 8 newly installed, 0 to remove and 85 not upgraded. 2022-03-08T18:32:13.607 INFO:teuthology.orchestra.run.smithi002.stdout:Need to get 73.6 MB of archives. 2022-03-08T18:32:13.607 INFO:teuthology.orchestra.run.smithi002.stdout:After this operation, 321 MB of additional disk space will be used. 2022-03-08T18:32:13.607 INFO:teuthology.orchestra.run.smithi002.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 conntrack amd64 1:1.4.5-2 [30.3 kB] 2022-03-08T18:32:13.671 INFO:teuthology.orchestra.run.smithi137.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2022-03-08T18:32:13.673 INFO:teuthology.orchestra.run.smithi066.stdout:Get:1 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 cri-tools amd64 1.19.0-00 [11.2 MB] 2022-03-08T18:32:13.705 INFO:teuthology.orchestra.run.smithi137.stdout:Get:2 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 kubernetes-cni amd64 0.8.7-00 [25.0 MB] 2022-03-08T18:32:13.723 INFO:teuthology.orchestra.run.smithi002.stdout:Get:1 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 cri-tools amd64 1.19.0-00 [11.2 MB] 2022-03-08T18:32:13.783 INFO:teuthology.orchestra.run.smithi066.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 ebtables amd64 2.0.11-3build1 [80.3 kB] 2022-03-08T18:32:13.786 INFO:teuthology.orchestra.run.smithi002.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 ebtables amd64 2.0.11-3build1 [80.3 kB] 2022-03-08T18:32:13.932 INFO:teuthology.orchestra.run.smithi002.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2022-03-08T18:32:13.966 INFO:teuthology.orchestra.run.smithi066.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2022-03-08T18:32:14.183 INFO:teuthology.orchestra.run.smithi066.stdout:Get:2 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 kubernetes-cni amd64 0.8.7-00 [25.0 MB] 2022-03-08T18:32:14.257 INFO:teuthology.orchestra.run.smithi137.stdout:Get:3 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 kubelet amd64 1.23.4-00 [19.5 MB] 2022-03-08T18:32:14.602 INFO:teuthology.orchestra.run.smithi002.stdout:Get:2 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 kubernetes-cni amd64 0.8.7-00 [25.0 MB] 2022-03-08T18:32:14.730 INFO:teuthology.orchestra.run.smithi137.stdout:Get:5 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 kubectl amd64 1.23.4-00 [8,927 kB] 2022-03-08T18:32:14.878 INFO:teuthology.orchestra.run.smithi137.stdout:Get:6 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 kubeadm amd64 1.23.4-00 [8,583 kB] 2022-03-08T18:32:14.977 INFO:teuthology.orchestra.run.smithi066.stdout:Get:3 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 kubelet amd64 1.23.4-00 [19.5 MB] 2022-03-08T18:32:15.311 INFO:teuthology.orchestra.run.smithi137.stderr:debconf: unable to initialize frontend: Dialog 2022-03-08T18:32:15.311 INFO:teuthology.orchestra.run.smithi137.stderr:debconf: (Dialog frontend will not work on a dumb terminal, an emacs shell buffer, or without a controlling terminal.) 2022-03-08T18:32:15.311 INFO:teuthology.orchestra.run.smithi137.stderr:debconf: falling back to frontend: Readline 2022-03-08T18:32:15.319 INFO:teuthology.orchestra.run.smithi137.stderr:debconf: unable to initialize frontend: Readline 2022-03-08T18:32:15.320 INFO:teuthology.orchestra.run.smithi137.stderr:debconf: (This frontend requires a controlling tty.) 2022-03-08T18:32:15.320 INFO:teuthology.orchestra.run.smithi137.stderr:debconf: falling back to frontend: Teletype 2022-03-08T18:32:15.325 INFO:teuthology.orchestra.run.smithi137.stderr:dpkg-preconfigure: unable to re-open stdin: 2022-03-08T18:32:15.376 INFO:teuthology.orchestra.run.smithi137.stdout:Fetched 73.6 MB in 2s (38.9 MB/s) 2022-03-08T18:32:15.463 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package conntrack. 2022-03-08T18:32:15.507 INFO:teuthology.orchestra.run.smithi137.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 109244 files and directories currently installed.) 2022-03-08T18:32:15.510 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../0-conntrack_1%3a1.4.5-2_amd64.deb ... 2022-03-08T18:32:15.523 INFO:teuthology.orchestra.run.smithi066.stdout:Get:5 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 kubectl amd64 1.23.4-00 [8,927 kB] 2022-03-08T18:32:15.554 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking conntrack (1:1.4.5-2) ... 2022-03-08T18:32:15.657 INFO:teuthology.orchestra.run.smithi002.stdout:Get:3 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 kubelet amd64 1.23.4-00 [19.5 MB] 2022-03-08T18:32:15.707 INFO:teuthology.orchestra.run.smithi066.stdout:Get:6 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 kubeadm amd64 1.23.4-00 [8,583 kB] 2022-03-08T18:32:15.848 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package cri-tools. 2022-03-08T18:32:15.862 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../1-cri-tools_1.19.0-00_amd64.deb ... 2022-03-08T18:32:15.907 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking cri-tools (1.19.0-00) ... 2022-03-08T18:32:16.144 INFO:teuthology.orchestra.run.smithi066.stderr:debconf: unable to initialize frontend: Dialog 2022-03-08T18:32:16.145 INFO:teuthology.orchestra.run.smithi066.stderr:debconf: (Dialog frontend will not work on a dumb terminal, an emacs shell buffer, or without a controlling terminal.) 2022-03-08T18:32:16.145 INFO:teuthology.orchestra.run.smithi066.stderr:debconf: falling back to frontend: Readline 2022-03-08T18:32:16.153 INFO:teuthology.orchestra.run.smithi066.stderr:debconf: unable to initialize frontend: Readline 2022-03-08T18:32:16.154 INFO:teuthology.orchestra.run.smithi066.stderr:debconf: (This frontend requires a controlling tty.) 2022-03-08T18:32:16.154 INFO:teuthology.orchestra.run.smithi066.stderr:debconf: falling back to frontend: Teletype 2022-03-08T18:32:16.159 INFO:teuthology.orchestra.run.smithi066.stderr:dpkg-preconfigure: unable to re-open stdin: 2022-03-08T18:32:16.207 INFO:teuthology.orchestra.run.smithi066.stdout:Fetched 73.6 MB in 3s (29.1 MB/s) 2022-03-08T18:32:16.234 INFO:teuthology.orchestra.run.smithi002.stdout:Get:5 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 kubectl amd64 1.23.4-00 [8,927 kB] 2022-03-08T18:32:16.302 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package conntrack. 2022-03-08T18:32:16.346 INFO:teuthology.orchestra.run.smithi066.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 109244 files and directories currently installed.) 2022-03-08T18:32:16.350 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../0-conntrack_1%3a1.4.5-2_amd64.deb ... 2022-03-08T18:32:16.353 INFO:teuthology.orchestra.run.smithi002.stdout:Get:6 https://packages.cloud.google.com/apt kubernetes-xenial/main amd64 kubeadm amd64 1.23.4-00 [8,583 kB] 2022-03-08T18:32:16.391 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking conntrack (1:1.4.5-2) ... 2022-03-08T18:32:16.704 INFO:teuthology.orchestra.run.smithi002.stderr:debconf: unable to initialize frontend: Dialog 2022-03-08T18:32:16.704 INFO:teuthology.orchestra.run.smithi002.stderr:debconf: (Dialog frontend will not work on a dumb terminal, an emacs shell buffer, or without a controlling terminal.) 2022-03-08T18:32:16.704 INFO:teuthology.orchestra.run.smithi002.stderr:debconf: falling back to frontend: Readline 2022-03-08T18:32:16.711 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package cri-tools. 2022-03-08T18:32:16.713 INFO:teuthology.orchestra.run.smithi002.stderr:debconf: unable to initialize frontend: Readline 2022-03-08T18:32:16.713 INFO:teuthology.orchestra.run.smithi002.stderr:debconf: (This frontend requires a controlling tty.) 2022-03-08T18:32:16.713 INFO:teuthology.orchestra.run.smithi002.stderr:debconf: falling back to frontend: Teletype 2022-03-08T18:32:16.718 INFO:teuthology.orchestra.run.smithi002.stderr:dpkg-preconfigure: unable to re-open stdin: 2022-03-08T18:32:16.726 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../1-cri-tools_1.19.0-00_amd64.deb ... 2022-03-08T18:32:16.769 INFO:teuthology.orchestra.run.smithi002.stdout:Fetched 73.6 MB in 3s (23.9 MB/s) 2022-03-08T18:32:16.769 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking cri-tools (1.19.0-00) ... 2022-03-08T18:32:16.842 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package conntrack. 2022-03-08T18:32:16.885 INFO:teuthology.orchestra.run.smithi002.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 109244 files and directories currently installed.) 2022-03-08T18:32:16.888 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../0-conntrack_1%3a1.4.5-2_amd64.deb ... 2022-03-08T18:32:16.931 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking conntrack (1:1.4.5-2) ... 2022-03-08T18:32:16.960 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ebtables. 2022-03-08T18:32:16.972 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../2-ebtables_2.0.11-3build1_amd64.deb ... 2022-03-08T18:32:17.024 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ebtables (2.0.11-3build1) ... 2022-03-08T18:32:17.267 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package cri-tools. 2022-03-08T18:32:17.283 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../1-cri-tools_1.19.0-00_amd64.deb ... 2022-03-08T18:32:17.287 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package kubernetes-cni. 2022-03-08T18:32:17.300 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../3-kubernetes-cni_0.8.7-00_amd64.deb ... 2022-03-08T18:32:17.326 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking cri-tools (1.19.0-00) ... 2022-03-08T18:32:17.337 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking kubernetes-cni (0.8.7-00) ... 2022-03-08T18:32:17.872 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ebtables. 2022-03-08T18:32:17.885 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../2-ebtables_2.0.11-3build1_amd64.deb ... 2022-03-08T18:32:17.938 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ebtables (2.0.11-3build1) ... 2022-03-08T18:32:18.217 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package kubernetes-cni. 2022-03-08T18:32:18.230 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../3-kubernetes-cni_0.8.7-00_amd64.deb ... 2022-03-08T18:32:18.267 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking kubernetes-cni (0.8.7-00) ... 2022-03-08T18:32:18.362 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ebtables. 2022-03-08T18:32:18.375 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../2-ebtables_2.0.11-3build1_amd64.deb ... 2022-03-08T18:32:18.529 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ebtables (2.0.11-3build1) ... 2022-03-08T18:32:18.832 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package socat. 2022-03-08T18:32:18.846 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../4-socat_1.7.3.3-2_amd64.deb ... 2022-03-08T18:32:18.848 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package kubernetes-cni. 2022-03-08T18:32:18.861 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../3-kubernetes-cni_0.8.7-00_amd64.deb ... 2022-03-08T18:32:18.889 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking socat (1.7.3.3-2) ... 2022-03-08T18:32:18.898 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking kubernetes-cni (0.8.7-00) ... 2022-03-08T18:32:19.251 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package kubelet. 2022-03-08T18:32:19.267 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../5-kubelet_1.23.4-00_amd64.deb ... 2022-03-08T18:32:19.326 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking kubelet (1.23.4-00) ... 2022-03-08T18:32:19.778 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package socat. 2022-03-08T18:32:19.791 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../4-socat_1.7.3.3-2_amd64.deb ... 2022-03-08T18:32:19.836 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking socat (1.7.3.3-2) ... 2022-03-08T18:32:20.264 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package kubelet. 2022-03-08T18:32:20.277 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../5-kubelet_1.23.4-00_amd64.deb ... 2022-03-08T18:32:20.309 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking kubelet (1.23.4-00) ... 2022-03-08T18:32:20.334 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package socat. 2022-03-08T18:32:20.346 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../4-socat_1.7.3.3-2_amd64.deb ... 2022-03-08T18:32:20.383 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking socat (1.7.3.3-2) ... 2022-03-08T18:32:20.779 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package kubelet. 2022-03-08T18:32:20.791 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../5-kubelet_1.23.4-00_amd64.deb ... 2022-03-08T18:32:20.828 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking kubelet (1.23.4-00) ... 2022-03-08T18:32:21.728 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package kubectl. 2022-03-08T18:32:21.742 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../6-kubectl_1.23.4-00_amd64.deb ... 2022-03-08T18:32:21.786 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking kubectl (1.23.4-00) ... 2022-03-08T18:32:22.695 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package kubectl. 2022-03-08T18:32:22.709 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../6-kubectl_1.23.4-00_amd64.deb ... 2022-03-08T18:32:22.753 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking kubectl (1.23.4-00) ... 2022-03-08T18:32:22.950 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package kubeadm. 2022-03-08T18:32:22.963 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../7-kubeadm_1.23.4-00_amd64.deb ... 2022-03-08T18:32:23.000 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking kubeadm (1.23.4-00) ... 2022-03-08T18:32:23.248 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package kubectl. 2022-03-08T18:32:23.263 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../6-kubectl_1.23.4-00_amd64.deb ... 2022-03-08T18:32:23.306 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking kubectl (1.23.4-00) ... 2022-03-08T18:32:24.005 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package kubeadm. 2022-03-08T18:32:24.019 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../7-kubeadm_1.23.4-00_amd64.deb ... 2022-03-08T18:32:24.064 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking kubeadm (1.23.4-00) ... 2022-03-08T18:32:24.103 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up conntrack (1:1.4.5-2) ... 2022-03-08T18:32:24.228 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up kubectl (1.23.4-00) ... 2022-03-08T18:32:24.355 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ebtables (2.0.11-3build1) ... 2022-03-08T18:32:24.500 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package kubeadm. 2022-03-08T18:32:24.515 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../7-kubeadm_1.23.4-00_amd64.deb ... 2022-03-08T18:32:24.558 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking kubeadm (1.23.4-00) ... 2022-03-08T18:32:25.127 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up socat (1.7.3.3-2) ... 2022-03-08T18:32:25.212 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up conntrack (1:1.4.5-2) ... 2022-03-08T18:32:25.252 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up cri-tools (1.19.0-00) ... 2022-03-08T18:32:25.345 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up kubectl (1.23.4-00) ... 2022-03-08T18:32:25.378 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up kubernetes-cni (0.8.7-00) ... 2022-03-08T18:32:25.496 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ebtables (2.0.11-3build1) ... 2022-03-08T18:32:25.498 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up kubelet (1.23.4-00) ... 2022-03-08T18:32:25.676 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up conntrack (1:1.4.5-2) ... 2022-03-08T18:32:25.714 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/kubelet.service -> /lib/systemd/system/kubelet.service. 2022-03-08T18:32:25.814 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up kubectl (1.23.4-00) ... 2022-03-08T18:32:25.924 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ebtables (2.0.11-3build1) ... 2022-03-08T18:32:26.193 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up kubeadm (1.23.4-00) ... 2022-03-08T18:32:26.327 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up socat (1.7.3.3-2) ... 2022-03-08T18:32:26.486 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up cri-tools (1.19.0-00) ... 2022-03-08T18:32:26.612 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up kubernetes-cni (0.8.7-00) ... 2022-03-08T18:32:26.713 INFO:teuthology.orchestra.run.smithi137.stdout:Processing triggers for man-db (2.9.1-1) ... 2022-03-08T18:32:26.714 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up kubelet (1.23.4-00) ... 2022-03-08T18:32:26.770 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up socat (1.7.3.3-2) ... 2022-03-08T18:32:26.898 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up cri-tools (1.19.0-00) ... 2022-03-08T18:32:26.921 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/kubelet.service -> /lib/systemd/system/kubelet.service. 2022-03-08T18:32:27.024 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up kubernetes-cni (0.8.7-00) ... 2022-03-08T18:32:27.142 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up kubelet (1.23.4-00) ... 2022-03-08T18:32:27.362 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/kubelet.service -> /lib/systemd/system/kubelet.service. 2022-03-08T18:32:27.431 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up kubeadm (1.23.4-00) ... 2022-03-08T18:32:27.842 INFO:teuthology.orchestra.run.smithi066.stdout:Processing triggers for man-db (2.9.1-1) ... 2022-03-08T18:32:27.851 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up kubeadm (1.23.4-00) ... 2022-03-08T18:32:28.328 INFO:teuthology.orchestra.run.smithi002.stdout:Processing triggers for man-db (2.9.1-1) ... 2022-03-08T18:32:29.242 DEBUG:teuthology.orchestra.run.smithi002:> sudo systemctl enable --now kubelet && sudo kubeadm config images pull 2022-03-08T18:32:29.293 DEBUG:teuthology.orchestra.run.smithi066:> sudo systemctl enable --now kubelet && sudo kubeadm config images pull 2022-03-08T18:32:29.295 DEBUG:teuthology.orchestra.run.smithi137:> sudo systemctl enable --now kubelet && sudo kubeadm config images pull 2022-03-08T18:32:41.022 INFO:teuthology.orchestra.run.smithi137.stdout:[config/images] Pulled k8s.gcr.io/kube-apiserver:v1.23.4 2022-03-08T18:32:42.086 INFO:teuthology.orchestra.run.smithi066.stdout:[config/images] Pulled k8s.gcr.io/kube-apiserver:v1.23.4 2022-03-08T18:32:42.643 INFO:teuthology.orchestra.run.smithi002.stdout:[config/images] Pulled k8s.gcr.io/kube-apiserver:v1.23.4 2022-03-08T18:32:45.132 INFO:teuthology.orchestra.run.smithi066.stdout:[config/images] Pulled k8s.gcr.io/kube-controller-manager:v1.23.4 2022-03-08T18:32:46.548 INFO:teuthology.orchestra.run.smithi137.stdout:[config/images] Pulled k8s.gcr.io/kube-controller-manager:v1.23.4 2022-03-08T18:32:46.946 INFO:teuthology.orchestra.run.smithi002.stdout:[config/images] Pulled k8s.gcr.io/kube-controller-manager:v1.23.4 2022-03-08T18:32:47.579 INFO:teuthology.orchestra.run.smithi066.stdout:[config/images] Pulled k8s.gcr.io/kube-scheduler:v1.23.4 2022-03-08T18:32:49.147 INFO:teuthology.orchestra.run.smithi137.stdout:[config/images] Pulled k8s.gcr.io/kube-scheduler:v1.23.4 2022-03-08T18:32:49.417 INFO:teuthology.orchestra.run.smithi002.stdout:[config/images] Pulled k8s.gcr.io/kube-scheduler:v1.23.4 2022-03-08T18:32:51.219 INFO:teuthology.orchestra.run.smithi066.stdout:[config/images] Pulled k8s.gcr.io/kube-proxy:v1.23.4 2022-03-08T18:32:52.586 INFO:teuthology.orchestra.run.smithi137.stdout:[config/images] Pulled k8s.gcr.io/kube-proxy:v1.23.4 2022-03-08T18:32:52.736 INFO:teuthology.orchestra.run.smithi066.stdout:[config/images] Pulled k8s.gcr.io/pause:3.6 2022-03-08T18:32:53.042 INFO:teuthology.orchestra.run.smithi002.stdout:[config/images] Pulled k8s.gcr.io/kube-proxy:v1.23.4 2022-03-08T18:32:54.103 INFO:teuthology.orchestra.run.smithi137.stdout:[config/images] Pulled k8s.gcr.io/pause:3.6 2022-03-08T18:32:54.750 INFO:teuthology.orchestra.run.smithi002.stdout:[config/images] Pulled k8s.gcr.io/pause:3.6 2022-03-08T18:32:59.127 INFO:teuthology.orchestra.run.smithi066.stdout:[config/images] Pulled k8s.gcr.io/etcd:3.5.1-0 2022-03-08T18:33:00.743 INFO:teuthology.orchestra.run.smithi137.stdout:[config/images] Pulled k8s.gcr.io/etcd:3.5.1-0 2022-03-08T18:33:01.041 INFO:teuthology.orchestra.run.smithi002.stdout:[config/images] Pulled k8s.gcr.io/etcd:3.5.1-0 2022-03-08T18:33:01.225 INFO:teuthology.orchestra.run.smithi066.stdout:[config/images] Pulled k8s.gcr.io/coredns/coredns:v1.8.6 2022-03-08T18:33:03.060 INFO:teuthology.orchestra.run.smithi137.stdout:[config/images] Pulled k8s.gcr.io/coredns/coredns:v1.8.6 2022-03-08T18:33:03.923 INFO:teuthology.orchestra.run.smithi002.stdout:[config/images] Pulled k8s.gcr.io/coredns/coredns:v1.8.6 2022-03-08T18:33:03.927 INFO:tasks.kubeadm:Token: abcdef.o6m5vxsd4tgghcfj 2022-03-08T18:33:03.927 INFO:tasks.kubeadm:Remotes: {Remote(name='ubuntu@smithi002.front.sepia.ceph.com'): '172.21.15.2', Remote(name='ubuntu@smithi066.front.sepia.ceph.com'): '172.21.15.66', Remote(name='ubuntu@smithi137.front.sepia.ceph.com'): '172.21.15.137'} 2022-03-08T18:33:03.928 DEBUG:teuthology.orchestra.run.smithi002:> sudo kubeadm init --node-name smithi002 --token abcdef.o6m5vxsd4tgghcfj --pod-network-cidr 10.248.8.0/21 2022-03-08T18:33:04.169 INFO:teuthology.orchestra.run.smithi002.stdout:[init] Using Kubernetes version: v1.23.4 2022-03-08T18:33:04.170 INFO:teuthology.orchestra.run.smithi002.stdout:[preflight] Running pre-flight checks 2022-03-08T18:33:04.587 INFO:teuthology.orchestra.run.smithi002.stdout:[preflight] Pulling images required for setting up a Kubernetes cluster 2022-03-08T18:33:04.587 INFO:teuthology.orchestra.run.smithi002.stdout:[preflight] This might take a minute or two, depending on the speed of your internet connection 2022-03-08T18:33:04.587 INFO:teuthology.orchestra.run.smithi002.stdout:[preflight] You can also perform this action in beforehand using 'kubeadm config images pull' 2022-03-08T18:33:04.894 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] Using certificateDir folder "/etc/kubernetes/pki" 2022-03-08T18:33:05.102 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] Generating "ca" certificate and key 2022-03-08T18:33:05.188 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] Generating "apiserver" certificate and key 2022-03-08T18:33:05.188 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] apiserver serving cert is signed for DNS names [kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local smithi002] and IPs [10.96.0.1 172.21.15.2] 2022-03-08T18:33:05.514 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] Generating "apiserver-kubelet-client" certificate and key 2022-03-08T18:33:05.625 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] Generating "front-proxy-ca" certificate and key 2022-03-08T18:33:05.963 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] Generating "front-proxy-client" certificate and key 2022-03-08T18:33:06.089 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] Generating "etcd/ca" certificate and key 2022-03-08T18:33:06.231 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] Generating "etcd/server" certificate and key 2022-03-08T18:33:06.231 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] etcd/server serving cert is signed for DNS names [localhost smithi002] and IPs [172.21.15.2 127.0.0.1 ::1] 2022-03-08T18:33:06.879 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] Generating "etcd/peer" certificate and key 2022-03-08T18:33:06.879 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] etcd/peer serving cert is signed for DNS names [localhost smithi002] and IPs [172.21.15.2 127.0.0.1 ::1] 2022-03-08T18:33:06.996 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] Generating "etcd/healthcheck-client" certificate and key 2022-03-08T18:33:07.201 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] Generating "apiserver-etcd-client" certificate and key 2022-03-08T18:33:07.438 INFO:teuthology.orchestra.run.smithi002.stdout:[certs] Generating "sa" key and public key 2022-03-08T18:33:07.439 INFO:teuthology.orchestra.run.smithi002.stdout:[kubeconfig] Using kubeconfig folder "/etc/kubernetes" 2022-03-08T18:33:07.630 INFO:teuthology.orchestra.run.smithi002.stdout:[kubeconfig] Writing "admin.conf" kubeconfig file 2022-03-08T18:33:07.908 INFO:teuthology.orchestra.run.smithi002.stdout:[kubeconfig] Writing "kubelet.conf" kubeconfig file 2022-03-08T18:33:08.301 INFO:teuthology.orchestra.run.smithi002.stdout:[kubeconfig] Writing "controller-manager.conf" kubeconfig file 2022-03-08T18:33:08.597 INFO:teuthology.orchestra.run.smithi002.stdout:[kubeconfig] Writing "scheduler.conf" kubeconfig file 2022-03-08T18:33:08.610 INFO:teuthology.orchestra.run.smithi002.stdout:[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env" 2022-03-08T18:33:08.611 INFO:teuthology.orchestra.run.smithi002.stdout:[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml" 2022-03-08T18:33:08.611 INFO:teuthology.orchestra.run.smithi002.stdout:[kubelet-start] Starting the kubelet 2022-03-08T18:33:08.819 INFO:teuthology.orchestra.run.smithi002.stdout:[control-plane] Using manifest folder "/etc/kubernetes/manifests" 2022-03-08T18:33:08.820 INFO:teuthology.orchestra.run.smithi002.stdout:[control-plane] Creating static Pod manifest for "kube-apiserver" 2022-03-08T18:33:08.823 INFO:teuthology.orchestra.run.smithi002.stdout:[control-plane] Creating static Pod manifest for "kube-controller-manager" 2022-03-08T18:33:08.825 INFO:teuthology.orchestra.run.smithi002.stdout:[control-plane] Creating static Pod manifest for "kube-scheduler" 2022-03-08T18:33:08.827 INFO:teuthology.orchestra.run.smithi002.stdout:[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests" 2022-03-08T18:33:08.827 INFO:teuthology.orchestra.run.smithi002.stdout:[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s 2022-03-08T18:33:25.831 INFO:teuthology.orchestra.run.smithi002.stdout:[apiclient] All control plane components are healthy after 17.003958 seconds 2022-03-08T18:33:25.831 INFO:teuthology.orchestra.run.smithi002.stdout:[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace 2022-03-08T18:33:25.848 INFO:teuthology.orchestra.run.smithi002.stdout:[kubelet] Creating a ConfigMap "kubelet-config-1.23" in namespace kube-system with the configuration for the kubelets in the cluster 2022-03-08T18:33:25.848 INFO:teuthology.orchestra.run.smithi002.stdout:NOTE: The "kubelet-config-1.23" naming of the kubelet ConfigMap is deprecated. Once the UnversionedKubeletConfigMap feature gate graduates to Beta the default name will become just "kubelet-config". Kubeadm upgrade will handle this transition transparently. 2022-03-08T18:33:26.440 INFO:teuthology.orchestra.run.smithi002.stdout:[upload-certs] Skipping phase. Please see --upload-certs 2022-03-08T18:33:26.441 INFO:teuthology.orchestra.run.smithi002.stdout:[mark-control-plane] Marking the node smithi002 as control-plane by adding the labels: [node-role.kubernetes.io/master(deprecated) node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers] 2022-03-08T18:33:26.441 INFO:teuthology.orchestra.run.smithi002.stdout:[mark-control-plane] Marking the node smithi002 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule] 2022-03-08T18:33:26.953 INFO:teuthology.orchestra.run.smithi002.stdout:[bootstrap-token] Using token: abcdef.o6m5vxsd4tgghcfj 2022-03-08T18:33:26.954 INFO:teuthology.orchestra.run.smithi002.stdout:[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles 2022-03-08T18:33:26.976 INFO:teuthology.orchestra.run.smithi002.stdout:[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes 2022-03-08T18:33:26.997 INFO:teuthology.orchestra.run.smithi002.stdout:[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials 2022-03-08T18:33:27.092 INFO:teuthology.orchestra.run.smithi002.stdout:[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token 2022-03-08T18:33:27.106 INFO:teuthology.orchestra.run.smithi002.stdout:[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster 2022-03-08T18:33:27.134 INFO:teuthology.orchestra.run.smithi002.stdout:[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace 2022-03-08T18:33:27.173 INFO:teuthology.orchestra.run.smithi002.stdout:[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key 2022-03-08T18:33:27.565 INFO:teuthology.orchestra.run.smithi002.stdout:[addons] Applied essential addon: CoreDNS 2022-03-08T18:33:27.689 INFO:teuthology.orchestra.run.smithi002.stdout:[addons] Applied essential addon: kube-proxy 2022-03-08T18:33:27.690 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:33:27.690 INFO:teuthology.orchestra.run.smithi002.stdout:Your Kubernetes control-plane has initialized successfully! 2022-03-08T18:33:27.691 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:33:27.691 INFO:teuthology.orchestra.run.smithi002.stdout:To start using your cluster, you need to run the following as a regular user: 2022-03-08T18:33:27.691 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:33:27.691 INFO:teuthology.orchestra.run.smithi002.stdout: mkdir -p $HOME/.kube 2022-03-08T18:33:27.692 INFO:teuthology.orchestra.run.smithi002.stdout: sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config 2022-03-08T18:33:27.692 INFO:teuthology.orchestra.run.smithi002.stdout: sudo chown $(id -u):$(id -g) $HOME/.kube/config 2022-03-08T18:33:27.692 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:33:27.692 INFO:teuthology.orchestra.run.smithi002.stdout:Alternatively, if you are the root user, you can run: 2022-03-08T18:33:27.692 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:33:27.693 INFO:teuthology.orchestra.run.smithi002.stdout: export KUBECONFIG=/etc/kubernetes/admin.conf 2022-03-08T18:33:27.693 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:33:27.693 INFO:teuthology.orchestra.run.smithi002.stdout:You should now deploy a pod network to the cluster. 2022-03-08T18:33:27.693 INFO:teuthology.orchestra.run.smithi002.stdout:Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: 2022-03-08T18:33:27.694 INFO:teuthology.orchestra.run.smithi002.stdout: https://kubernetes.io/docs/concepts/cluster-administration/addons/ 2022-03-08T18:33:27.694 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:33:27.694 INFO:teuthology.orchestra.run.smithi002.stdout:Then you can join any number of worker nodes by running the following on each as root: 2022-03-08T18:33:27.694 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:33:27.694 INFO:teuthology.orchestra.run.smithi002.stdout:kubeadm join 172.21.15.2:6443 --token abcdef.o6m5vxsd4tgghcfj \ 2022-03-08T18:33:27.695 INFO:teuthology.orchestra.run.smithi002.stdout: --discovery-token-ca-cert-hash sha256:36b0218de30d858a5bca6155130a1d959ea436fb8da42b61afb46b75cda6a075 2022-03-08T18:33:27.696 DEBUG:teuthology.orchestra.run.smithi066:> sudo kubeadm join 172.21.15.2:6443 --node-name smithi066 --token abcdef.o6m5vxsd4tgghcfj --discovery-token-unsafe-skip-ca-verification 2022-03-08T18:33:27.699 DEBUG:teuthology.orchestra.run.smithi137:> sudo kubeadm join 172.21.15.2:6443 --node-name smithi137 --token abcdef.o6m5vxsd4tgghcfj --discovery-token-unsafe-skip-ca-verification 2022-03-08T18:33:27.744 INFO:teuthology.orchestra.run.smithi066.stdout:[preflight] Running pre-flight checks 2022-03-08T18:33:27.744 INFO:teuthology.orchestra.run.smithi137.stdout:[preflight] Running pre-flight checks 2022-03-08T18:33:46.527 INFO:teuthology.orchestra.run.smithi137.stdout:[preflight] Reading configuration from the cluster... 2022-03-08T18:33:46.528 INFO:teuthology.orchestra.run.smithi137.stdout:[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml' 2022-03-08T18:33:46.528 INFO:teuthology.orchestra.run.smithi066.stdout:[preflight] Reading configuration from the cluster... 2022-03-08T18:33:46.529 INFO:teuthology.orchestra.run.smithi066.stdout:[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml' 2022-03-08T18:33:46.879 INFO:teuthology.orchestra.run.smithi066.stderr:W0308 18:33:46.877845 10669 utils.go:69] The recommended value for "resolvConf" in "KubeletConfiguration" is: /run/systemd/resolve/resolv.conf; the provided value is: /run/systemd/resolve/resolv.conf 2022-03-08T18:33:46.879 INFO:teuthology.orchestra.run.smithi137.stderr:W0308 18:33:46.879829 10671 utils.go:69] The recommended value for "resolvConf" in "KubeletConfiguration" is: /run/systemd/resolve/resolv.conf; the provided value is: /run/systemd/resolve/resolv.conf 2022-03-08T18:33:46.898 INFO:teuthology.orchestra.run.smithi137.stdout:[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml" 2022-03-08T18:33:46.899 INFO:teuthology.orchestra.run.smithi137.stdout:[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env" 2022-03-08T18:33:46.899 INFO:teuthology.orchestra.run.smithi137.stdout:[kubelet-start] Starting the kubelet 2022-03-08T18:33:46.900 INFO:teuthology.orchestra.run.smithi066.stdout:[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml" 2022-03-08T18:33:46.900 INFO:teuthology.orchestra.run.smithi066.stdout:[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env" 2022-03-08T18:33:46.900 INFO:teuthology.orchestra.run.smithi066.stdout:[kubelet-start] Starting the kubelet 2022-03-08T18:33:47.109 INFO:teuthology.orchestra.run.smithi137.stdout:[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap... 2022-03-08T18:33:47.144 INFO:teuthology.orchestra.run.smithi066.stdout:[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap... 2022-03-08T18:33:52.635 INFO:teuthology.orchestra.run.smithi137.stdout: 2022-03-08T18:33:52.635 INFO:teuthology.orchestra.run.smithi137.stdout:This node has joined the cluster: 2022-03-08T18:33:52.635 INFO:teuthology.orchestra.run.smithi137.stdout:* Certificate signing request was sent to apiserver and a response was received. 2022-03-08T18:33:52.635 INFO:teuthology.orchestra.run.smithi137.stdout:* The Kubelet was informed of the new secure connection details. 2022-03-08T18:33:52.636 INFO:teuthology.orchestra.run.smithi137.stdout: 2022-03-08T18:33:52.636 INFO:teuthology.orchestra.run.smithi137.stdout:Run 'kubectl get nodes' on the control-plane to see this node join the cluster. 2022-03-08T18:33:52.636 INFO:teuthology.orchestra.run.smithi137.stdout: 2022-03-08T18:33:52.674 INFO:teuthology.orchestra.run.smithi066.stdout: 2022-03-08T18:33:52.674 INFO:teuthology.orchestra.run.smithi066.stdout:This node has joined the cluster: 2022-03-08T18:33:52.675 INFO:teuthology.orchestra.run.smithi066.stdout:* Certificate signing request was sent to apiserver and a response was received. 2022-03-08T18:33:52.675 INFO:teuthology.orchestra.run.smithi066.stdout:* The Kubelet was informed of the new secure connection details. 2022-03-08T18:33:52.675 INFO:teuthology.orchestra.run.smithi066.stdout: 2022-03-08T18:33:52.675 INFO:teuthology.orchestra.run.smithi066.stdout:Run 'kubectl get nodes' on the control-plane to see this node join the cluster. 2022-03-08T18:33:52.676 INFO:teuthology.orchestra.run.smithi066.stdout: 2022-03-08T18:33:52.682 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:33:52.682 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd if=/etc/kubernetes/admin.conf of=/dev/stdout 2022-03-08T18:33:52.699 INFO:tasks.kubeadm:Setting up kubectl 2022-03-08T18:33:52.699 DEBUG:teuthology.orchestra.run.smithi002:> mkdir -p .kube && sudo mkdir -p /root/.kube 2022-03-08T18:33:52.756 DEBUG:teuthology.orchestra.run.smithi066:> mkdir -p .kube && sudo mkdir -p /root/.kube 2022-03-08T18:33:52.774 DEBUG:teuthology.orchestra.run.smithi137:> mkdir -p .kube && sudo mkdir -p /root/.kube 2022-03-08T18:33:52.792 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:33:52.792 DEBUG:teuthology.orchestra.run.smithi002:> dd of=.kube/config 2022-03-08T18:33:52.799 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:33:52.800 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd of=/root/.kube/config 2022-03-08T18:33:52.814 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:33:52.814 DEBUG:teuthology.orchestra.run.smithi066:> dd of=.kube/config 2022-03-08T18:33:52.823 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:33:52.823 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/root/.kube/config 2022-03-08T18:33:52.837 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:33:52.838 DEBUG:teuthology.orchestra.run.smithi137:> dd of=.kube/config 2022-03-08T18:33:52.845 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:33:52.846 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd of=/root/.kube/config 2022-03-08T18:33:52.902 DEBUG:teuthology.orchestra.run.smithi002:> kubectl apply -f https://docs.projectcalico.org/manifests/tigera-operator.yaml 2022-03-08T18:33:55.436 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/bgpconfigurations.crd.projectcalico.org created 2022-03-08T18:33:55.469 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/bgppeers.crd.projectcalico.org created 2022-03-08T18:33:55.539 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/blockaffinities.crd.projectcalico.org created 2022-03-08T18:33:55.610 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/caliconodestatuses.crd.projectcalico.org created 2022-03-08T18:33:55.702 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/clusterinformations.crd.projectcalico.org created 2022-03-08T18:33:55.734 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/felixconfigurations.crd.projectcalico.org created 2022-03-08T18:33:55.769 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/globalnetworkpolicies.crd.projectcalico.org created 2022-03-08T18:33:55.872 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/globalnetworksets.crd.projectcalico.org created 2022-03-08T18:33:55.903 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/hostendpoints.crd.projectcalico.org created 2022-03-08T18:33:55.954 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/ipamblocks.crd.projectcalico.org created 2022-03-08T18:33:56.046 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/ipamconfigs.crd.projectcalico.org created 2022-03-08T18:33:56.078 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/ipamhandles.crd.projectcalico.org created 2022-03-08T18:33:56.113 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/ippools.crd.projectcalico.org created 2022-03-08T18:33:56.179 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/ipreservations.crd.projectcalico.org created 2022-03-08T18:33:56.207 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/kubecontrollersconfigurations.crd.projectcalico.org created 2022-03-08T18:33:56.247 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/networkpolicies.crd.projectcalico.org created 2022-03-08T18:33:56.262 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/networksets.crd.projectcalico.org created 2022-03-08T18:33:56.354 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/apiservers.operator.tigera.io created 2022-03-08T18:33:56.367 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/imagesets.operator.tigera.io created 2022-03-08T18:33:56.423 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/installations.operator.tigera.io created 2022-03-08T18:33:56.545 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/tigerastatuses.operator.tigera.io created 2022-03-08T18:33:56.621 INFO:teuthology.orchestra.run.smithi002.stdout:namespace/tigera-operator created 2022-03-08T18:33:56.627 INFO:teuthology.orchestra.run.smithi002.stderr:Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ 2022-03-08T18:33:56.705 INFO:teuthology.orchestra.run.smithi002.stdout:podsecuritypolicy.policy/tigera-operator created 2022-03-08T18:33:56.712 INFO:teuthology.orchestra.run.smithi002.stdout:serviceaccount/tigera-operator created 2022-03-08T18:33:56.748 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/tigera-operator created 2022-03-08T18:33:56.857 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/tigera-operator created 2022-03-08T18:33:56.875 INFO:teuthology.orchestra.run.smithi002.stdout:deployment.apps/tigera-operator created 2022-03-08T18:33:56.881 DEBUG:teuthology.orchestra.run.smithi002:> kubectl create -f - 2022-03-08T18:34:00.059 INFO:teuthology.orchestra.run.smithi002.stdout:installation.operator.tigera.io/default created 2022-03-08T18:34:00.066 INFO:tasks.kubeadm:Scanning for scratch devices 2022-03-08T18:34:00.067 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:34:00.067 DEBUG:teuthology.orchestra.run.smithi002:> dd if=/scratch_devs of=/dev/stdout 2022-03-08T18:34:00.072 INFO:tasks.kubeadm:Scratch devices on smithi002: ['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2022-03-08T18:34:00.073 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 2022-03-08T18:34:00.094 INFO:teuthology.orchestra.run.smithi002.stderr:10+0 records in 2022-03-08T18:34:00.094 INFO:teuthology.orchestra.run.smithi002.stderr:10+0 records out 2022-03-08T18:34:00.094 INFO:teuthology.orchestra.run.smithi002.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0121459 s, 863 MB/s 2022-03-08T18:34:00.095 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 2022-03-08T18:34:00.116 INFO:teuthology.orchestra.run.smithi002.stderr:10+0 records in 2022-03-08T18:34:00.117 INFO:teuthology.orchestra.run.smithi002.stderr:10+0 records out 2022-03-08T18:34:00.117 INFO:teuthology.orchestra.run.smithi002.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0110466 s, 949 MB/s 2022-03-08T18:34:00.118 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 2022-03-08T18:34:00.181 INFO:teuthology.orchestra.run.smithi002.stderr:10+0 records in 2022-03-08T18:34:00.182 INFO:teuthology.orchestra.run.smithi002.stderr:10+0 records out 2022-03-08T18:34:00.182 INFO:teuthology.orchestra.run.smithi002.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0130505 s, 803 MB/s 2022-03-08T18:34:00.183 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 2022-03-08T18:34:00.206 INFO:teuthology.orchestra.run.smithi002.stderr:10+0 records in 2022-03-08T18:34:00.206 INFO:teuthology.orchestra.run.smithi002.stderr:10+0 records out 2022-03-08T18:34:00.207 INFO:teuthology.orchestra.run.smithi002.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0128657 s, 815 MB/s 2022-03-08T18:34:00.208 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:34:00.208 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/scratch_devs of=/dev/stdout 2022-03-08T18:34:00.216 INFO:tasks.kubeadm:Scratch devices on smithi066: ['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2022-03-08T18:34:00.217 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 2022-03-08T18:34:00.291 INFO:teuthology.orchestra.run.smithi066.stderr:10+0 records in 2022-03-08T18:34:00.291 INFO:teuthology.orchestra.run.smithi066.stderr:10+0 records out 2022-03-08T18:34:00.292 INFO:teuthology.orchestra.run.smithi066.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0128513 s, 816 MB/s 2022-03-08T18:34:00.293 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 2022-03-08T18:34:00.318 INFO:teuthology.orchestra.run.smithi066.stderr:10+0 records in 2022-03-08T18:34:00.319 INFO:teuthology.orchestra.run.smithi066.stderr:10+0 records out 2022-03-08T18:34:00.319 INFO:teuthology.orchestra.run.smithi066.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0153558 s, 683 MB/s 2022-03-08T18:34:00.320 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 2022-03-08T18:34:00.346 INFO:teuthology.orchestra.run.smithi066.stderr:10+0 records in 2022-03-08T18:34:00.347 INFO:teuthology.orchestra.run.smithi066.stderr:10+0 records out 2022-03-08T18:34:00.347 INFO:teuthology.orchestra.run.smithi066.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0145181 s, 722 MB/s 2022-03-08T18:34:00.348 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 2022-03-08T18:34:00.379 INFO:teuthology.orchestra.run.smithi066.stderr:10+0 records in 2022-03-08T18:34:00.379 INFO:teuthology.orchestra.run.smithi066.stderr:10+0 records out 2022-03-08T18:34:00.379 INFO:teuthology.orchestra.run.smithi066.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0163881 s, 640 MB/s 2022-03-08T18:34:00.381 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:34:00.381 DEBUG:teuthology.orchestra.run.smithi137:> dd if=/scratch_devs of=/dev/stdout 2022-03-08T18:34:00.389 INFO:tasks.kubeadm:Scratch devices on smithi137: ['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2022-03-08T18:34:00.389 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 2022-03-08T18:34:00.413 INFO:teuthology.orchestra.run.smithi137.stderr:10+0 records in 2022-03-08T18:34:00.413 INFO:teuthology.orchestra.run.smithi137.stderr:10+0 records out 2022-03-08T18:34:00.413 INFO:teuthology.orchestra.run.smithi137.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0140512 s, 746 MB/s 2022-03-08T18:34:00.414 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 2022-03-08T18:34:00.495 INFO:teuthology.orchestra.run.smithi137.stderr:10+0 records in 2022-03-08T18:34:00.496 INFO:teuthology.orchestra.run.smithi137.stderr:10+0 records out 2022-03-08T18:34:00.496 INFO:teuthology.orchestra.run.smithi137.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0168166 s, 624 MB/s 2022-03-08T18:34:00.496 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 2022-03-08T18:34:00.524 INFO:teuthology.orchestra.run.smithi137.stderr:10+0 records in 2022-03-08T18:34:00.524 INFO:teuthology.orchestra.run.smithi137.stderr:10+0 records out 2022-03-08T18:34:00.525 INFO:teuthology.orchestra.run.smithi137.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0154077 s, 681 MB/s 2022-03-08T18:34:00.526 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 2022-03-08T18:34:00.552 INFO:teuthology.orchestra.run.smithi137.stderr:10+0 records in 2022-03-08T18:34:00.552 INFO:teuthology.orchestra.run.smithi137.stderr:10+0 records out 2022-03-08T18:34:00.553 INFO:teuthology.orchestra.run.smithi137.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0158993 s, 660 MB/s 2022-03-08T18:34:00.571 INFO:tasks.kubeadm:Creating PVs + StorageClass 2022-03-08T18:34:00.571 DEBUG:tasks.kubeadm:apiVersion: v1 kind: PersistentVolume metadata: name: smithi002-lv-1 spec: accessModes: - ReadWriteOnce capacity: storage: 100Gi local: path: /dev/vg_nvme/lv_1 nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - smithi002 persistentVolumeReclaimPolicy: Retain storageClassName: scratch volumeMode: Block --- apiVersion: v1 kind: PersistentVolume metadata: name: smithi002-lv-2 spec: accessModes: - ReadWriteOnce capacity: storage: 100Gi local: path: /dev/vg_nvme/lv_2 nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - smithi002 persistentVolumeReclaimPolicy: Retain storageClassName: scratch volumeMode: Block --- apiVersion: v1 kind: PersistentVolume metadata: name: smithi002-lv-3 spec: accessModes: - ReadWriteOnce capacity: storage: 100Gi local: path: /dev/vg_nvme/lv_3 nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - smithi002 persistentVolumeReclaimPolicy: Retain storageClassName: scratch volumeMode: Block --- apiVersion: v1 kind: PersistentVolume metadata: name: smithi002-lv-4 spec: accessModes: - ReadWriteOnce capacity: storage: 100Gi local: path: /dev/vg_nvme/lv_4 nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - smithi002 persistentVolumeReclaimPolicy: Retain storageClassName: scratch volumeMode: Block --- apiVersion: v1 kind: PersistentVolume metadata: name: smithi066-lv-1 spec: accessModes: - ReadWriteOnce capacity: storage: 100Gi local: path: /dev/vg_nvme/lv_1 nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - smithi066 persistentVolumeReclaimPolicy: Retain storageClassName: scratch volumeMode: Block --- apiVersion: v1 kind: PersistentVolume metadata: name: smithi066-lv-2 spec: accessModes: - ReadWriteOnce capacity: storage: 100Gi local: path: /dev/vg_nvme/lv_2 nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - smithi066 persistentVolumeReclaimPolicy: Retain storageClassName: scratch volumeMode: Block --- apiVersion: v1 kind: PersistentVolume metadata: name: smithi066-lv-3 spec: accessModes: - ReadWriteOnce capacity: storage: 100Gi local: path: /dev/vg_nvme/lv_3 nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - smithi066 persistentVolumeReclaimPolicy: Retain storageClassName: scratch volumeMode: Block --- apiVersion: v1 kind: PersistentVolume metadata: name: smithi066-lv-4 spec: accessModes: - ReadWriteOnce capacity: storage: 100Gi local: path: /dev/vg_nvme/lv_4 nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - smithi066 persistentVolumeReclaimPolicy: Retain storageClassName: scratch volumeMode: Block --- apiVersion: v1 kind: PersistentVolume metadata: name: smithi137-lv-1 spec: accessModes: - ReadWriteOnce capacity: storage: 100Gi local: path: /dev/vg_nvme/lv_1 nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - smithi137 persistentVolumeReclaimPolicy: Retain storageClassName: scratch volumeMode: Block --- apiVersion: v1 kind: PersistentVolume metadata: name: smithi137-lv-2 spec: accessModes: - ReadWriteOnce capacity: storage: 100Gi local: path: /dev/vg_nvme/lv_2 nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - smithi137 persistentVolumeReclaimPolicy: Retain storageClassName: scratch volumeMode: Block --- apiVersion: v1 kind: PersistentVolume metadata: name: smithi137-lv-3 spec: accessModes: - ReadWriteOnce capacity: storage: 100Gi local: path: /dev/vg_nvme/lv_3 nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - smithi137 persistentVolumeReclaimPolicy: Retain storageClassName: scratch volumeMode: Block --- apiVersion: v1 kind: PersistentVolume metadata: name: smithi137-lv-4 spec: accessModes: - ReadWriteOnce capacity: storage: 100Gi local: path: /dev/vg_nvme/lv_4 nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - smithi137 persistentVolumeReclaimPolicy: Retain storageClassName: scratch volumeMode: Block --- apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: scratch provisioner: kubernetes.io/no-provisioner volumeBindingMode: WaitForFirstConsumer 2022-03-08T18:34:00.572 DEBUG:teuthology.orchestra.run.smithi002:> kubectl create -f - 2022-03-08T18:34:01.439 INFO:teuthology.orchestra.run.smithi002.stdout:persistentvolume/smithi002-lv-1 created 2022-03-08T18:34:01.443 INFO:teuthology.orchestra.run.smithi002.stdout:persistentvolume/smithi002-lv-2 created 2022-03-08T18:34:01.451 INFO:teuthology.orchestra.run.smithi002.stdout:persistentvolume/smithi002-lv-3 created 2022-03-08T18:34:01.460 INFO:teuthology.orchestra.run.smithi002.stdout:persistentvolume/smithi002-lv-4 created 2022-03-08T18:34:01.487 INFO:teuthology.orchestra.run.smithi002.stdout:persistentvolume/smithi066-lv-1 created 2022-03-08T18:34:01.494 INFO:teuthology.orchestra.run.smithi002.stdout:persistentvolume/smithi066-lv-2 created 2022-03-08T18:34:01.545 INFO:teuthology.orchestra.run.smithi002.stdout:persistentvolume/smithi066-lv-3 created 2022-03-08T18:34:01.552 INFO:teuthology.orchestra.run.smithi002.stdout:persistentvolume/smithi066-lv-4 created 2022-03-08T18:34:01.560 INFO:teuthology.orchestra.run.smithi002.stdout:persistentvolume/smithi137-lv-1 created 2022-03-08T18:34:01.569 INFO:teuthology.orchestra.run.smithi002.stdout:persistentvolume/smithi137-lv-2 created 2022-03-08T18:34:01.595 INFO:teuthology.orchestra.run.smithi002.stdout:persistentvolume/smithi137-lv-3 created 2022-03-08T18:34:01.602 INFO:teuthology.orchestra.run.smithi002.stdout:persistentvolume/smithi137-lv-4 created 2022-03-08T18:34:01.698 INFO:teuthology.orchestra.run.smithi002.stdout:storageclass.storage.k8s.io/scratch created 2022-03-08T18:34:01.702 DEBUG:teuthology.orchestra.run.smithi002:> kubectl taint node smithi002 node-role.kubernetes.io/master- || true 2022-03-08T18:34:01.774 INFO:teuthology.orchestra.run.smithi002.stdout:node/smithi002 untainted 2022-03-08T18:34:01.776 INFO:tasks.kubeadm:Kubeadm complete, yielding 2022-03-08T18:34:01.777 INFO:teuthology.run_tasks:Running task nvme_loop... 2022-03-08T18:34:01.789 INFO:tasks.nvme_loop:Setting up nvme_loop on scratch devices... 2022-03-08T18:34:01.789 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:34:01.790 DEBUG:teuthology.orchestra.run.smithi002:> dd if=/scratch_devs of=/dev/stdout 2022-03-08T18:34:01.797 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2022-03-08T18:34:01.798 DEBUG:teuthology.orchestra.run.smithi002:> stat /dev/vg_nvme/lv_1 && sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_1 2022-03-08T18:34:01.805 INFO:teuthology.orchestra.run.smithi002.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2022-03-08T18:34:01.805 INFO:teuthology.orchestra.run.smithi002.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-03-08T18:34:01.806 INFO:teuthology.orchestra.run.smithi002.stdout:Device: 5h/5d Inode: 499 Links: 1 2022-03-08T18:34:01.806 INFO:teuthology.orchestra.run.smithi002.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-03-08T18:34:01.806 INFO:teuthology.orchestra.run.smithi002.stdout:Access: 2022-03-08 18:34:00.096530011 +0000 2022-03-08T18:34:01.806 INFO:teuthology.orchestra.run.smithi002.stdout:Modify: 2022-03-08 18:34:00.096530011 +0000 2022-03-08T18:34:01.807 INFO:teuthology.orchestra.run.smithi002.stdout:Change: 2022-03-08 18:34:00.096530011 +0000 2022-03-08T18:34:01.807 INFO:teuthology.orchestra.run.smithi002.stdout: Birth: - 2022-03-08T18:34:01.813 INFO:teuthology.orchestra.run.smithi002.stderr:1+0 records in 2022-03-08T18:34:01.813 INFO:teuthology.orchestra.run.smithi002.stderr:1+0 records out 2022-03-08T18:34:01.814 INFO:teuthology.orchestra.run.smithi002.stderr:512 bytes copied, 0.000311446 s, 1.6 MB/s 2022-03-08T18:34:01.818 DEBUG:teuthology.orchestra.run.smithi002:> stat /dev/vg_nvme/lv_2 && sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_2 2022-03-08T18:34:01.824 INFO:teuthology.orchestra.run.smithi002.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2022-03-08T18:34:01.824 INFO:teuthology.orchestra.run.smithi002.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-03-08T18:34:01.824 INFO:teuthology.orchestra.run.smithi002.stdout:Device: 5h/5d Inode: 504 Links: 1 2022-03-08T18:34:01.825 INFO:teuthology.orchestra.run.smithi002.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-03-08T18:34:01.825 INFO:teuthology.orchestra.run.smithi002.stdout:Access: 2022-03-08 18:34:00.120530012 +0000 2022-03-08T18:34:01.825 INFO:teuthology.orchestra.run.smithi002.stdout:Modify: 2022-03-08 18:34:00.120530012 +0000 2022-03-08T18:34:01.825 INFO:teuthology.orchestra.run.smithi002.stdout:Change: 2022-03-08 18:34:00.120530012 +0000 2022-03-08T18:34:01.826 INFO:teuthology.orchestra.run.smithi002.stdout: Birth: - 2022-03-08T18:34:01.832 INFO:teuthology.orchestra.run.smithi002.stderr:1+0 records in 2022-03-08T18:34:01.833 INFO:teuthology.orchestra.run.smithi002.stderr:1+0 records out 2022-03-08T18:34:01.833 INFO:teuthology.orchestra.run.smithi002.stderr:512 bytes copied, 0.000295255 s, 1.7 MB/s 2022-03-08T18:34:01.837 DEBUG:teuthology.orchestra.run.smithi002:> stat /dev/vg_nvme/lv_3 && sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_3 2022-03-08T18:34:01.887 INFO:teuthology.orchestra.run.smithi002.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2022-03-08T18:34:01.887 INFO:teuthology.orchestra.run.smithi002.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-03-08T18:34:01.887 INFO:teuthology.orchestra.run.smithi002.stdout:Device: 5h/5d Inode: 510 Links: 1 2022-03-08T18:34:01.887 INFO:teuthology.orchestra.run.smithi002.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-03-08T18:34:01.888 INFO:teuthology.orchestra.run.smithi002.stdout:Access: 2022-03-08 18:34:00.184530013 +0000 2022-03-08T18:34:01.888 INFO:teuthology.orchestra.run.smithi002.stdout:Modify: 2022-03-08 18:34:00.184530013 +0000 2022-03-08T18:34:01.889 INFO:teuthology.orchestra.run.smithi002.stdout:Change: 2022-03-08 18:34:00.184530013 +0000 2022-03-08T18:34:01.889 INFO:teuthology.orchestra.run.smithi002.stdout: Birth: - 2022-03-08T18:34:01.895 INFO:teuthology.orchestra.run.smithi002.stderr:1+0 records in 2022-03-08T18:34:01.896 INFO:teuthology.orchestra.run.smithi002.stderr:1+0 records out 2022-03-08T18:34:01.896 INFO:teuthology.orchestra.run.smithi002.stderr:512 bytes copied, 0.000302398 s, 1.7 MB/s 2022-03-08T18:34:01.899 DEBUG:teuthology.orchestra.run.smithi002:> stat /dev/vg_nvme/lv_4 && sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_4 2022-03-08T18:34:01.905 INFO:teuthology.orchestra.run.smithi002.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2022-03-08T18:34:01.906 INFO:teuthology.orchestra.run.smithi002.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-03-08T18:34:01.906 INFO:teuthology.orchestra.run.smithi002.stdout:Device: 5h/5d Inode: 514 Links: 1 2022-03-08T18:34:01.906 INFO:teuthology.orchestra.run.smithi002.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-03-08T18:34:01.907 INFO:teuthology.orchestra.run.smithi002.stdout:Access: 2022-03-08 18:34:00.208530013 +0000 2022-03-08T18:34:01.907 INFO:teuthology.orchestra.run.smithi002.stdout:Modify: 2022-03-08 18:34:00.208530013 +0000 2022-03-08T18:34:01.907 INFO:teuthology.orchestra.run.smithi002.stdout:Change: 2022-03-08 18:34:00.208530013 +0000 2022-03-08T18:34:01.907 INFO:teuthology.orchestra.run.smithi002.stdout: Birth: - 2022-03-08T18:34:01.915 INFO:teuthology.orchestra.run.smithi002.stderr:1+0 records in 2022-03-08T18:34:01.915 INFO:teuthology.orchestra.run.smithi002.stderr:1+0 records out 2022-03-08T18:34:01.916 INFO:teuthology.orchestra.run.smithi002.stderr:512 bytes copied, 0.000199976 s, 2.6 MB/s 2022-03-08T18:34:01.919 DEBUG:teuthology.orchestra.run.smithi002:> sudo modprobe nvme_loop && sudo mkdir -p /sys/kernel/config/nvmet/hosts/hostnqn && sudo mkdir -p /sys/kernel/config/nvmet/ports/1 && echo loop | sudo tee /sys/kernel/config/nvmet/ports/1/addr_trtype 2022-03-08T18:34:02.012 INFO:teuthology.orchestra.run.smithi002.stdout:loop 2022-03-08T18:34:02.013 INFO:tasks.nvme_loop:Connecting nvme_loop smithi002:/dev/vg_nvme/lv_1... 2022-03-08T18:34:02.014 DEBUG:teuthology.orchestra.run.smithi002:> sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_1 && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_1/attr_allow_any_host && sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_1/namespaces/1 && echo -n /dev/vg_nvme/lv_1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_1/namespaces/1/device_path && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_1/namespaces/1/enable && sudo ln -s /sys/kernel/config/nvmet/subsystems/lv_1 /sys/kernel/config/nvmet/ports/1/subsystems/lv_1 && sudo nvme connect -t loop -n lv_1 -q hostnqn 2022-03-08T18:34:02.035 INFO:teuthology.orchestra.run.smithi002.stdout:1 2022-03-08T18:34:02.057 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/vg_nvme/lv_11 2022-03-08T18:34:02.079 INFO:tasks.nvme_loop:Connecting nvme_loop smithi002:/dev/vg_nvme/lv_2... 2022-03-08T18:34:02.080 DEBUG:teuthology.orchestra.run.smithi002:> sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_2 && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_2/attr_allow_any_host && sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_2/namespaces/1 && echo -n /dev/vg_nvme/lv_2 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_2/namespaces/1/device_path && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_2/namespaces/1/enable && sudo ln -s /sys/kernel/config/nvmet/subsystems/lv_2 /sys/kernel/config/nvmet/ports/1/subsystems/lv_2 && sudo nvme connect -t loop -n lv_2 -q hostnqn 2022-03-08T18:34:02.099 INFO:teuthology.orchestra.run.smithi002.stdout:1 2022-03-08T18:34:02.124 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/vg_nvme/lv_21 2022-03-08T18:34:02.142 INFO:tasks.nvme_loop:Connecting nvme_loop smithi002:/dev/vg_nvme/lv_3... 2022-03-08T18:34:02.142 DEBUG:teuthology.orchestra.run.smithi002:> sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_3 && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_3/attr_allow_any_host && sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_3/namespaces/1 && echo -n /dev/vg_nvme/lv_3 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_3/namespaces/1/device_path && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_3/namespaces/1/enable && sudo ln -s /sys/kernel/config/nvmet/subsystems/lv_3 /sys/kernel/config/nvmet/ports/1/subsystems/lv_3 && sudo nvme connect -t loop -n lv_3 -q hostnqn 2022-03-08T18:34:02.164 INFO:teuthology.orchestra.run.smithi002.stdout:1 2022-03-08T18:34:02.190 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/vg_nvme/lv_31 2022-03-08T18:34:02.214 INFO:tasks.nvme_loop:Connecting nvme_loop smithi002:/dev/vg_nvme/lv_4... 2022-03-08T18:34:02.215 DEBUG:teuthology.orchestra.run.smithi002:> sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_4 && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_4/attr_allow_any_host && sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_4/namespaces/1 && echo -n /dev/vg_nvme/lv_4 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_4/namespaces/1/device_path && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_4/namespaces/1/enable && sudo ln -s /sys/kernel/config/nvmet/subsystems/lv_4 /sys/kernel/config/nvmet/ports/1/subsystems/lv_4 && sudo nvme connect -t loop -n lv_4 -q hostnqn 2022-03-08T18:34:02.282 INFO:teuthology.orchestra.run.smithi002.stdout:1 2022-03-08T18:34:02.308 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/vg_nvme/lv_41 2022-03-08T18:34:02.331 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:34:02.332 DEBUG:teuthology.orchestra.run.smithi002:> dd if=/scratch_devs of=/dev/stdout 2022-03-08T18:34:02.337 DEBUG:teuthology.orchestra.run.smithi002:> sudo nvme list 2022-03-08T18:34:02.348 INFO:teuthology.orchestra.run.smithi002.stdout:Node SN Model Namespace Usage Format FW Rev 2022-03-08T18:34:02.349 INFO:teuthology.orchestra.run.smithi002.stdout:---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- 2022-03-08T18:34:02.349 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/nvme0n1 PHMB751300EV480DGN INTEL SSDPED1D480GA 1 480.10 GB / 480.10 GB 512 B + 0 B E2010325 2022-03-08T18:34:02.349 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/nvme1n1 a83cfd3cdd137c38 Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:02.350 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/nvme2n1 263f5d21702901ed Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:02.350 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/nvme3n1 c1a5ab378bc90d68 Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:02.351 INFO:tasks.nvme_loop:new_devs ['/dev/nvme1n1', '/dev/nvme2n1', '/dev/nvme3n1'] 2022-03-08T18:34:03.352 DEBUG:teuthology.orchestra.run.smithi002:> sudo nvme list 2022-03-08T18:34:03.367 INFO:teuthology.orchestra.run.smithi002.stdout:Node SN Model Namespace Usage Format FW Rev 2022-03-08T18:34:03.368 INFO:teuthology.orchestra.run.smithi002.stdout:---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- 2022-03-08T18:34:03.368 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/nvme0n1 PHMB751300EV480DGN INTEL SSDPED1D480GA 1 480.10 GB / 480.10 GB 512 B + 0 B E2010325 2022-03-08T18:34:03.368 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/nvme1n1 a83cfd3cdd137c38 Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:03.368 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/nvme2n1 263f5d21702901ed Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:03.369 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/nvme3n1 c1a5ab378bc90d68 Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:03.369 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/nvme4n1 027a2b0497639839 Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:03.370 INFO:tasks.nvme_loop:new_devs ['/dev/nvme1n1', '/dev/nvme2n1', '/dev/nvme3n1', '/dev/nvme4n1'] 2022-03-08T18:34:03.370 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:34:03.371 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd of=/scratch_devs 2022-03-08T18:34:03.429 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:34:03.430 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/scratch_devs of=/dev/stdout 2022-03-08T18:34:03.437 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2022-03-08T18:34:03.438 DEBUG:teuthology.orchestra.run.smithi066:> stat /dev/vg_nvme/lv_1 && sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_1 2022-03-08T18:34:03.444 INFO:teuthology.orchestra.run.smithi066.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2022-03-08T18:34:03.444 INFO:teuthology.orchestra.run.smithi066.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-03-08T18:34:03.445 INFO:teuthology.orchestra.run.smithi066.stdout:Device: 5h/5d Inode: 502 Links: 1 2022-03-08T18:34:03.445 INFO:teuthology.orchestra.run.smithi066.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-03-08T18:34:03.445 INFO:teuthology.orchestra.run.smithi066.stdout:Access: 2022-03-08 18:34:00.294131950 +0000 2022-03-08T18:34:03.445 INFO:teuthology.orchestra.run.smithi066.stdout:Modify: 2022-03-08 18:34:00.290131950 +0000 2022-03-08T18:34:03.446 INFO:teuthology.orchestra.run.smithi066.stdout:Change: 2022-03-08 18:34:00.290131950 +0000 2022-03-08T18:34:03.446 INFO:teuthology.orchestra.run.smithi066.stdout: Birth: - 2022-03-08T18:34:03.452 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records in 2022-03-08T18:34:03.453 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records out 2022-03-08T18:34:03.453 INFO:teuthology.orchestra.run.smithi066.stderr:512 bytes copied, 0.000565737 s, 905 kB/s 2022-03-08T18:34:03.456 DEBUG:teuthology.orchestra.run.smithi066:> stat /dev/vg_nvme/lv_2 && sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_2 2022-03-08T18:34:03.463 INFO:teuthology.orchestra.run.smithi066.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2022-03-08T18:34:03.463 INFO:teuthology.orchestra.run.smithi066.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-03-08T18:34:03.463 INFO:teuthology.orchestra.run.smithi066.stdout:Device: 5h/5d Inode: 508 Links: 1 2022-03-08T18:34:03.464 INFO:teuthology.orchestra.run.smithi066.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-03-08T18:34:03.464 INFO:teuthology.orchestra.run.smithi066.stdout:Access: 2022-03-08 18:34:00.322131951 +0000 2022-03-08T18:34:03.464 INFO:teuthology.orchestra.run.smithi066.stdout:Modify: 2022-03-08 18:34:00.318131951 +0000 2022-03-08T18:34:03.464 INFO:teuthology.orchestra.run.smithi066.stdout:Change: 2022-03-08 18:34:00.318131951 +0000 2022-03-08T18:34:03.464 INFO:teuthology.orchestra.run.smithi066.stdout: Birth: - 2022-03-08T18:34:03.470 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records in 2022-03-08T18:34:03.471 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records out 2022-03-08T18:34:03.471 INFO:teuthology.orchestra.run.smithi066.stderr:512 bytes copied, 0.000418343 s, 1.2 MB/s 2022-03-08T18:34:03.475 DEBUG:teuthology.orchestra.run.smithi066:> stat /dev/vg_nvme/lv_3 && sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_3 2022-03-08T18:34:03.523 INFO:teuthology.orchestra.run.smithi066.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2022-03-08T18:34:03.523 INFO:teuthology.orchestra.run.smithi066.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-03-08T18:34:03.524 INFO:teuthology.orchestra.run.smithi066.stdout:Device: 5h/5d Inode: 513 Links: 1 2022-03-08T18:34:03.524 INFO:teuthology.orchestra.run.smithi066.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-03-08T18:34:03.524 INFO:teuthology.orchestra.run.smithi066.stdout:Access: 2022-03-08 18:34:00.350131952 +0000 2022-03-08T18:34:03.524 INFO:teuthology.orchestra.run.smithi066.stdout:Modify: 2022-03-08 18:34:00.350131952 +0000 2022-03-08T18:34:03.525 INFO:teuthology.orchestra.run.smithi066.stdout:Change: 2022-03-08 18:34:00.350131952 +0000 2022-03-08T18:34:03.525 INFO:teuthology.orchestra.run.smithi066.stdout: Birth: - 2022-03-08T18:34:03.533 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records in 2022-03-08T18:34:03.533 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records out 2022-03-08T18:34:03.534 INFO:teuthology.orchestra.run.smithi066.stderr:512 bytes copied, 0.000574701 s, 891 kB/s 2022-03-08T18:34:03.537 DEBUG:teuthology.orchestra.run.smithi066:> stat /dev/vg_nvme/lv_4 && sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_4 2022-03-08T18:34:03.544 INFO:teuthology.orchestra.run.smithi066.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2022-03-08T18:34:03.544 INFO:teuthology.orchestra.run.smithi066.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-03-08T18:34:03.545 INFO:teuthology.orchestra.run.smithi066.stdout:Device: 5h/5d Inode: 520 Links: 1 2022-03-08T18:34:03.545 INFO:teuthology.orchestra.run.smithi066.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-03-08T18:34:03.545 INFO:teuthology.orchestra.run.smithi066.stdout:Access: 2022-03-08 18:34:00.378131953 +0000 2022-03-08T18:34:03.545 INFO:teuthology.orchestra.run.smithi066.stdout:Modify: 2022-03-08 18:34:00.378131953 +0000 2022-03-08T18:34:03.546 INFO:teuthology.orchestra.run.smithi066.stdout:Change: 2022-03-08 18:34:00.378131953 +0000 2022-03-08T18:34:03.546 INFO:teuthology.orchestra.run.smithi066.stdout: Birth: - 2022-03-08T18:34:03.553 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records in 2022-03-08T18:34:03.554 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records out 2022-03-08T18:34:03.554 INFO:teuthology.orchestra.run.smithi066.stderr:512 bytes copied, 0.000488367 s, 1.0 MB/s 2022-03-08T18:34:03.558 DEBUG:teuthology.orchestra.run.smithi066:> sudo modprobe nvme_loop && sudo mkdir -p /sys/kernel/config/nvmet/hosts/hostnqn && sudo mkdir -p /sys/kernel/config/nvmet/ports/1 && echo loop | sudo tee /sys/kernel/config/nvmet/ports/1/addr_trtype 2022-03-08T18:34:03.644 INFO:teuthology.orchestra.run.smithi066.stdout:loop 2022-03-08T18:34:03.646 INFO:tasks.nvme_loop:Connecting nvme_loop smithi066:/dev/vg_nvme/lv_1... 2022-03-08T18:34:03.646 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_1 && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_1/attr_allow_any_host && sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_1/namespaces/1 && echo -n /dev/vg_nvme/lv_1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_1/namespaces/1/device_path && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_1/namespaces/1/enable && sudo ln -s /sys/kernel/config/nvmet/subsystems/lv_1 /sys/kernel/config/nvmet/ports/1/subsystems/lv_1 && sudo nvme connect -t loop -n lv_1 -q hostnqn 2022-03-08T18:34:03.669 INFO:teuthology.orchestra.run.smithi066.stdout:1 2022-03-08T18:34:03.695 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/vg_nvme/lv_11 2022-03-08T18:34:03.717 INFO:tasks.nvme_loop:Connecting nvme_loop smithi066:/dev/vg_nvme/lv_2... 2022-03-08T18:34:03.719 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_2 && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_2/attr_allow_any_host && sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_2/namespaces/1 && echo -n /dev/vg_nvme/lv_2 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_2/namespaces/1/device_path && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_2/namespaces/1/enable && sudo ln -s /sys/kernel/config/nvmet/subsystems/lv_2 /sys/kernel/config/nvmet/ports/1/subsystems/lv_2 && sudo nvme connect -t loop -n lv_2 -q hostnqn 2022-03-08T18:34:03.745 INFO:teuthology.orchestra.run.smithi066.stdout:1 2022-03-08T18:34:03.770 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/vg_nvme/lv_21 2022-03-08T18:34:03.795 INFO:tasks.nvme_loop:Connecting nvme_loop smithi066:/dev/vg_nvme/lv_3... 2022-03-08T18:34:03.796 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_3 && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_3/attr_allow_any_host && sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_3/namespaces/1 && echo -n /dev/vg_nvme/lv_3 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_3/namespaces/1/device_path && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_3/namespaces/1/enable && sudo ln -s /sys/kernel/config/nvmet/subsystems/lv_3 /sys/kernel/config/nvmet/ports/1/subsystems/lv_3 && sudo nvme connect -t loop -n lv_3 -q hostnqn 2022-03-08T18:34:03.861 INFO:teuthology.orchestra.run.smithi066.stdout:1 2022-03-08T18:34:03.886 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/vg_nvme/lv_31 2022-03-08T18:34:03.908 INFO:tasks.nvme_loop:Connecting nvme_loop smithi066:/dev/vg_nvme/lv_4... 2022-03-08T18:34:03.909 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_4 && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_4/attr_allow_any_host && sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_4/namespaces/1 && echo -n /dev/vg_nvme/lv_4 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_4/namespaces/1/device_path && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_4/namespaces/1/enable && sudo ln -s /sys/kernel/config/nvmet/subsystems/lv_4 /sys/kernel/config/nvmet/ports/1/subsystems/lv_4 && sudo nvme connect -t loop -n lv_4 -q hostnqn 2022-03-08T18:34:03.927 INFO:teuthology.orchestra.run.smithi066.stdout:1 2022-03-08T18:34:03.955 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/vg_nvme/lv_41 2022-03-08T18:34:03.977 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:34:03.977 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/scratch_devs of=/dev/stdout 2022-03-08T18:34:03.984 DEBUG:teuthology.orchestra.run.smithi066:> sudo nvme list 2022-03-08T18:34:04.040 INFO:teuthology.orchestra.run.smithi066.stdout:Node SN Model Namespace Usage Format FW Rev 2022-03-08T18:34:04.040 INFO:teuthology.orchestra.run.smithi066.stdout:---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- 2022-03-08T18:34:04.041 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/nvme0n1 CVFT53430091400BGN INTEL SSDPEDMD400G4 1 400.09 GB / 400.09 GB 512 B + 0 B 8DV101H0 2022-03-08T18:34:04.041 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/nvme1n1 280cf0cd4ef73e9d Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:04.041 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/nvme2n1 65997c71bdd7c9c7 Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:04.041 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/nvme3n1 c27c74537a64bed5 Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:04.041 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/nvme4n1 0ff3dc2bf30dd309 Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:04.043 INFO:tasks.nvme_loop:new_devs ['/dev/nvme1n1', '/dev/nvme2n1', '/dev/nvme3n1', '/dev/nvme4n1'] 2022-03-08T18:34:04.043 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:34:04.043 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/scratch_devs 2022-03-08T18:34:04.056 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:34:04.056 DEBUG:teuthology.orchestra.run.smithi137:> dd if=/scratch_devs of=/dev/stdout 2022-03-08T18:34:04.064 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2022-03-08T18:34:04.065 DEBUG:teuthology.orchestra.run.smithi137:> stat /dev/vg_nvme/lv_1 && sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_1 2022-03-08T18:34:04.072 INFO:teuthology.orchestra.run.smithi137.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2022-03-08T18:34:04.073 INFO:teuthology.orchestra.run.smithi137.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-03-08T18:34:04.073 INFO:teuthology.orchestra.run.smithi137.stdout:Device: 5h/5d Inode: 500 Links: 1 2022-03-08T18:34:04.073 INFO:teuthology.orchestra.run.smithi137.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-03-08T18:34:04.073 INFO:teuthology.orchestra.run.smithi137.stdout:Access: 2022-03-08 18:34:00.412184195 +0000 2022-03-08T18:34:04.074 INFO:teuthology.orchestra.run.smithi137.stdout:Modify: 2022-03-08 18:34:00.412184195 +0000 2022-03-08T18:34:04.074 INFO:teuthology.orchestra.run.smithi137.stdout:Change: 2022-03-08 18:34:00.412184195 +0000 2022-03-08T18:34:04.074 INFO:teuthology.orchestra.run.smithi137.stdout: Birth: - 2022-03-08T18:34:04.084 INFO:teuthology.orchestra.run.smithi137.stderr:1+0 records in 2022-03-08T18:34:04.084 INFO:teuthology.orchestra.run.smithi137.stderr:1+0 records out 2022-03-08T18:34:04.084 INFO:teuthology.orchestra.run.smithi137.stderr:512 bytes copied, 0.000550846 s, 929 kB/s 2022-03-08T18:34:04.088 DEBUG:teuthology.orchestra.run.smithi137:> stat /dev/vg_nvme/lv_2 && sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_2 2022-03-08T18:34:04.095 INFO:teuthology.orchestra.run.smithi137.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2022-03-08T18:34:04.095 INFO:teuthology.orchestra.run.smithi137.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-03-08T18:34:04.095 INFO:teuthology.orchestra.run.smithi137.stdout:Device: 5h/5d Inode: 507 Links: 1 2022-03-08T18:34:04.095 INFO:teuthology.orchestra.run.smithi137.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-03-08T18:34:04.096 INFO:teuthology.orchestra.run.smithi137.stdout:Access: 2022-03-08 18:34:00.496184194 +0000 2022-03-08T18:34:04.096 INFO:teuthology.orchestra.run.smithi137.stdout:Modify: 2022-03-08 18:34:00.492184194 +0000 2022-03-08T18:34:04.097 INFO:teuthology.orchestra.run.smithi137.stdout:Change: 2022-03-08 18:34:00.492184194 +0000 2022-03-08T18:34:04.097 INFO:teuthology.orchestra.run.smithi137.stdout: Birth: - 2022-03-08T18:34:04.103 INFO:teuthology.orchestra.run.smithi137.stderr:1+0 records in 2022-03-08T18:34:04.103 INFO:teuthology.orchestra.run.smithi137.stderr:1+0 records out 2022-03-08T18:34:04.104 INFO:teuthology.orchestra.run.smithi137.stderr:512 bytes copied, 0.00054178 s, 945 kB/s 2022-03-08T18:34:04.108 DEBUG:teuthology.orchestra.run.smithi137:> stat /dev/vg_nvme/lv_3 && sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_3 2022-03-08T18:34:04.116 INFO:teuthology.orchestra.run.smithi137.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2022-03-08T18:34:04.116 INFO:teuthology.orchestra.run.smithi137.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-03-08T18:34:04.116 INFO:teuthology.orchestra.run.smithi137.stdout:Device: 5h/5d Inode: 511 Links: 1 2022-03-08T18:34:04.117 INFO:teuthology.orchestra.run.smithi137.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-03-08T18:34:04.117 INFO:teuthology.orchestra.run.smithi137.stdout:Access: 2022-03-08 18:34:00.528184193 +0000 2022-03-08T18:34:04.117 INFO:teuthology.orchestra.run.smithi137.stdout:Modify: 2022-03-08 18:34:00.528184193 +0000 2022-03-08T18:34:04.117 INFO:teuthology.orchestra.run.smithi137.stdout:Change: 2022-03-08 18:34:00.528184193 +0000 2022-03-08T18:34:04.118 INFO:teuthology.orchestra.run.smithi137.stdout: Birth: - 2022-03-08T18:34:04.124 INFO:teuthology.orchestra.run.smithi137.stderr:1+0 records in 2022-03-08T18:34:04.125 INFO:teuthology.orchestra.run.smithi137.stderr:1+0 records out 2022-03-08T18:34:04.125 INFO:teuthology.orchestra.run.smithi137.stderr:512 bytes copied, 0.000534001 s, 959 kB/s 2022-03-08T18:34:04.129 DEBUG:teuthology.orchestra.run.smithi137:> stat /dev/vg_nvme/lv_4 && sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_4 2022-03-08T18:34:04.177 INFO:teuthology.orchestra.run.smithi137.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2022-03-08T18:34:04.178 INFO:teuthology.orchestra.run.smithi137.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-03-08T18:34:04.178 INFO:teuthology.orchestra.run.smithi137.stdout:Device: 5h/5d Inode: 520 Links: 1 2022-03-08T18:34:04.178 INFO:teuthology.orchestra.run.smithi137.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-03-08T18:34:04.179 INFO:teuthology.orchestra.run.smithi137.stdout:Access: 2022-03-08 18:34:00.556184192 +0000 2022-03-08T18:34:04.179 INFO:teuthology.orchestra.run.smithi137.stdout:Modify: 2022-03-08 18:34:00.552184192 +0000 2022-03-08T18:34:04.179 INFO:teuthology.orchestra.run.smithi137.stdout:Change: 2022-03-08 18:34:00.552184192 +0000 2022-03-08T18:34:04.179 INFO:teuthology.orchestra.run.smithi137.stdout: Birth: - 2022-03-08T18:34:04.187 INFO:teuthology.orchestra.run.smithi137.stderr:1+0 records in 2022-03-08T18:34:04.187 INFO:teuthology.orchestra.run.smithi137.stderr:1+0 records out 2022-03-08T18:34:04.188 INFO:teuthology.orchestra.run.smithi137.stderr:512 bytes copied, 0.000535753 s, 956 kB/s 2022-03-08T18:34:04.191 DEBUG:teuthology.orchestra.run.smithi137:> sudo modprobe nvme_loop && sudo mkdir -p /sys/kernel/config/nvmet/hosts/hostnqn && sudo mkdir -p /sys/kernel/config/nvmet/ports/1 && echo loop | sudo tee /sys/kernel/config/nvmet/ports/1/addr_trtype 2022-03-08T18:34:04.336 INFO:teuthology.orchestra.run.smithi137.stdout:loop 2022-03-08T18:34:04.337 INFO:tasks.nvme_loop:Connecting nvme_loop smithi137:/dev/vg_nvme/lv_1... 2022-03-08T18:34:04.337 DEBUG:teuthology.orchestra.run.smithi137:> sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_1 && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_1/attr_allow_any_host && sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_1/namespaces/1 && echo -n /dev/vg_nvme/lv_1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_1/namespaces/1/device_path && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_1/namespaces/1/enable && sudo ln -s /sys/kernel/config/nvmet/subsystems/lv_1 /sys/kernel/config/nvmet/ports/1/subsystems/lv_1 && sudo nvme connect -t loop -n lv_1 -q hostnqn 2022-03-08T18:34:04.359 INFO:teuthology.orchestra.run.smithi137.stdout:1 2022-03-08T18:34:04.381 INFO:teuthology.orchestra.run.smithi137.stdout:/dev/vg_nvme/lv_11 2022-03-08T18:34:04.401 INFO:tasks.nvme_loop:Connecting nvme_loop smithi137:/dev/vg_nvme/lv_2... 2022-03-08T18:34:04.401 DEBUG:teuthology.orchestra.run.smithi137:> sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_2 && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_2/attr_allow_any_host && sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_2/namespaces/1 && echo -n /dev/vg_nvme/lv_2 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_2/namespaces/1/device_path && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_2/namespaces/1/enable && sudo ln -s /sys/kernel/config/nvmet/subsystems/lv_2 /sys/kernel/config/nvmet/ports/1/subsystems/lv_2 && sudo nvme connect -t loop -n lv_2 -q hostnqn 2022-03-08T18:34:04.421 INFO:teuthology.orchestra.run.smithi137.stdout:1 2022-03-08T18:34:04.445 INFO:teuthology.orchestra.run.smithi137.stdout:/dev/vg_nvme/lv_21 2022-03-08T18:34:04.464 INFO:tasks.nvme_loop:Connecting nvme_loop smithi137:/dev/vg_nvme/lv_3... 2022-03-08T18:34:04.465 DEBUG:teuthology.orchestra.run.smithi137:> sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_3 && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_3/attr_allow_any_host && sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_3/namespaces/1 && echo -n /dev/vg_nvme/lv_3 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_3/namespaces/1/device_path && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_3/namespaces/1/enable && sudo ln -s /sys/kernel/config/nvmet/subsystems/lv_3 /sys/kernel/config/nvmet/ports/1/subsystems/lv_3 && sudo nvme connect -t loop -n lv_3 -q hostnqn 2022-03-08T18:34:04.534 INFO:teuthology.orchestra.run.smithi137.stdout:1 2022-03-08T18:34:04.558 INFO:teuthology.orchestra.run.smithi137.stdout:/dev/vg_nvme/lv_31 2022-03-08T18:34:04.579 INFO:tasks.nvme_loop:Connecting nvme_loop smithi137:/dev/vg_nvme/lv_4... 2022-03-08T18:34:04.579 DEBUG:teuthology.orchestra.run.smithi137:> sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_4 && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_4/attr_allow_any_host && sudo mkdir -p /sys/kernel/config/nvmet/subsystems/lv_4/namespaces/1 && echo -n /dev/vg_nvme/lv_4 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_4/namespaces/1/device_path && echo 1 | sudo tee /sys/kernel/config/nvmet/subsystems/lv_4/namespaces/1/enable && sudo ln -s /sys/kernel/config/nvmet/subsystems/lv_4 /sys/kernel/config/nvmet/ports/1/subsystems/lv_4 && sudo nvme connect -t loop -n lv_4 -q hostnqn 2022-03-08T18:34:04.599 INFO:teuthology.orchestra.run.smithi137.stdout:1 2022-03-08T18:34:04.625 INFO:teuthology.orchestra.run.smithi137.stdout:/dev/vg_nvme/lv_41 2022-03-08T18:34:04.650 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:34:04.650 DEBUG:teuthology.orchestra.run.smithi137:> dd if=/scratch_devs of=/dev/stdout 2022-03-08T18:34:04.697 DEBUG:teuthology.orchestra.run.smithi137:> sudo nvme list 2022-03-08T18:34:04.709 INFO:teuthology.orchestra.run.smithi137.stdout:Node SN Model Namespace Usage Format FW Rev 2022-03-08T18:34:04.709 INFO:teuthology.orchestra.run.smithi137.stdout:---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- 2022-03-08T18:34:04.710 INFO:teuthology.orchestra.run.smithi137.stdout:/dev/nvme0n1 CVFT6233004A400BGN INTEL SSDPEDMD400G4 1 400.09 GB / 400.09 GB 512 B + 0 B 8DV101H0 2022-03-08T18:34:04.710 INFO:teuthology.orchestra.run.smithi137.stdout:/dev/nvme1n1 87b57356e846d2ed Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:04.710 INFO:teuthology.orchestra.run.smithi137.stdout:/dev/nvme2n1 f5ce12f335a6b3cb Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:04.712 INFO:teuthology.orchestra.run.smithi137.stdout:/dev/nvme3n1 d5b218afaf21226d Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:04.712 INFO:teuthology.orchestra.run.smithi137.stdout:/dev/nvme4n1 19080e3d8b24bc6c Linux 1 96.00 GB / 96.00 GB 512 B + 0 B 5.13.0-3 2022-03-08T18:34:04.714 INFO:tasks.nvme_loop:new_devs ['/dev/nvme1n1', '/dev/nvme2n1', '/dev/nvme3n1', '/dev/nvme4n1'] 2022-03-08T18:34:04.714 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:34:04.715 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd of=/scratch_devs 2022-03-08T18:34:04.729 INFO:teuthology.run_tasks:Running task rook... 2022-03-08T18:34:04.803 INFO:tasks.rook:Rook start 2022-03-08T18:34:04.804 INFO:tasks.rook:Config: {'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-whitelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '6fe8e680a334072b1ba19b2de006862f98aedbae', 'rook_branch': 'v1.7.2', 'rook_image': 'rook/ceph:v1.7.2'} 2022-03-08T18:34:04.805 INFO:tasks.rook:Ceph image is quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae 2022-03-08T18:34:04.805 INFO:tasks.rook:Cloning https://github.com/rook/rook branch v1.7.2 2022-03-08T18:34:04.805 DEBUG:teuthology.orchestra.run.smithi002:> rm -rf rook && git clone --single-branch --branch v1.7.2 https://github.com/rook/rook rook 2022-03-08T18:34:04.814 INFO:teuthology.orchestra.run.smithi002.stderr:Cloning into 'rook'... 2022-03-08T18:34:06.960 INFO:teuthology.orchestra.run.smithi002.stderr:Note: switching to '1784489964d2e417b6050c7edc8f626b5cca49f5'. 2022-03-08T18:34:06.960 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:34:06.961 INFO:teuthology.orchestra.run.smithi002.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2022-03-08T18:34:06.961 INFO:teuthology.orchestra.run.smithi002.stderr:changes and commit them, and you can discard any commits you make in this 2022-03-08T18:34:06.961 INFO:teuthology.orchestra.run.smithi002.stderr:state without impacting any branches by switching back to a branch. 2022-03-08T18:34:06.961 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:34:06.961 INFO:teuthology.orchestra.run.smithi002.stderr:If you want to create a new branch to retain commits you create, you may 2022-03-08T18:34:06.962 INFO:teuthology.orchestra.run.smithi002.stderr:do so (now or later) by using -c with the switch command. Example: 2022-03-08T18:34:06.962 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:34:06.962 INFO:teuthology.orchestra.run.smithi002.stderr: git switch -c 2022-03-08T18:34:06.962 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:34:06.963 INFO:teuthology.orchestra.run.smithi002.stderr:Or undo this operation with: 2022-03-08T18:34:06.963 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:34:06.963 INFO:teuthology.orchestra.run.smithi002.stderr: git switch - 2022-03-08T18:34:06.963 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:34:06.964 INFO:teuthology.orchestra.run.smithi002.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2022-03-08T18:34:06.964 INFO:teuthology.orchestra.run.smithi002.stderr: 2022-03-08T18:34:07.063 INFO:tasks.rook:/home/teuthworker 2022-03-08T18:34:07.064 DEBUG:teuthology.orchestra.run.smithi002:> PATH=/sbin:/usr/sbin ip addr show 2022-03-08T18:34:07.072 INFO:teuthology.orchestra.run.smithi002.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2022-03-08T18:34:07.072 INFO:teuthology.orchestra.run.smithi002.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2022-03-08T18:34:07.073 INFO:teuthology.orchestra.run.smithi002.stdout: inet 127.0.0.1/8 scope host lo 2022-03-08T18:34:07.073 INFO:teuthology.orchestra.run.smithi002.stdout: valid_lft forever preferred_lft forever 2022-03-08T18:34:07.073 INFO:teuthology.orchestra.run.smithi002.stdout: inet6 ::1/128 scope host 2022-03-08T18:34:07.073 INFO:teuthology.orchestra.run.smithi002.stdout: valid_lft forever preferred_lft forever 2022-03-08T18:34:07.074 INFO:teuthology.orchestra.run.smithi002.stdout:2: eno1: mtu 1500 qdisc noop state DOWN group default qlen 1000 2022-03-08T18:34:07.074 INFO:teuthology.orchestra.run.smithi002.stdout: link/ether 0c:c4:7a:6b:fe:02 brd ff:ff:ff:ff:ff:ff 2022-03-08T18:34:07.074 INFO:teuthology.orchestra.run.smithi002.stdout: altname enp7s0f0 2022-03-08T18:34:07.074 INFO:teuthology.orchestra.run.smithi002.stdout:3: eno2: mtu 1500 qdisc noop state DOWN group default qlen 1000 2022-03-08T18:34:07.074 INFO:teuthology.orchestra.run.smithi002.stdout: link/ether 0c:c4:7a:6b:fe:03 brd ff:ff:ff:ff:ff:ff 2022-03-08T18:34:07.075 INFO:teuthology.orchestra.run.smithi002.stdout: altname enp7s0f1 2022-03-08T18:34:07.075 INFO:teuthology.orchestra.run.smithi002.stdout:4: ens1f0: mtu 1500 qdisc mq state UP group default qlen 1000 2022-03-08T18:34:07.075 INFO:teuthology.orchestra.run.smithi002.stdout: link/ether 0c:c4:7a:bd:19:20 brd ff:ff:ff:ff:ff:ff 2022-03-08T18:34:07.075 INFO:teuthology.orchestra.run.smithi002.stdout: altname enp1s0f0 2022-03-08T18:34:07.076 INFO:teuthology.orchestra.run.smithi002.stdout: inet 172.21.15.2/20 brd 172.21.15.255 scope global dynamic ens1f0 2022-03-08T18:34:07.076 INFO:teuthology.orchestra.run.smithi002.stdout: valid_lft 42828sec preferred_lft 42828sec 2022-03-08T18:34:07.076 INFO:teuthology.orchestra.run.smithi002.stdout: inet6 fe80::ec4:7aff:febd:1920/64 scope link 2022-03-08T18:34:07.076 INFO:teuthology.orchestra.run.smithi002.stdout: valid_lft forever preferred_lft forever 2022-03-08T18:34:07.077 INFO:teuthology.orchestra.run.smithi002.stdout:5: ens1f1: mtu 1500 qdisc noop state DOWN group default qlen 1000 2022-03-08T18:34:07.077 INFO:teuthology.orchestra.run.smithi002.stdout: link/ether 0c:c4:7a:bd:19:21 brd ff:ff:ff:ff:ff:ff 2022-03-08T18:34:07.077 INFO:teuthology.orchestra.run.smithi002.stdout: altname enp1s0f1 2022-03-08T18:34:07.077 INFO:teuthology.orchestra.run.smithi002.stdout:6: docker0: mtu 1500 qdisc noqueue state DOWN group default 2022-03-08T18:34:07.077 INFO:teuthology.orchestra.run.smithi002.stdout: link/ether 02:42:57:74:47:b0 brd ff:ff:ff:ff:ff:ff 2022-03-08T18:34:07.078 INFO:teuthology.orchestra.run.smithi002.stdout: inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0 2022-03-08T18:34:07.078 INFO:teuthology.orchestra.run.smithi002.stdout: valid_lft forever preferred_lft forever 2022-03-08T18:34:07.079 DEBUG:teuthology.orchestra.run.smithi002:> which systemctl 2022-03-08T18:34:07.125 INFO:teuthology.orchestra.run.smithi002.stdout:/usr/bin/systemctl 2022-03-08T18:34:07.126 DEBUG:teuthology.orchestra.run.smithi002:> test -f /run/.containerenv -o -f /.dockerenv 2022-03-08T18:34:07.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-03-08T18:34:07.132 DEBUG:teuthology.orchestra.run.smithi002:> true 2022-03-08T18:34:07.164 DEBUG:teuthology.orchestra.run.smithi002:> sudo lsb_release -is 2022-03-08T18:34:07.214 INFO:teuthology.orchestra.run.smithi002.stdout:Ubuntu 2022-03-08T18:34:07.220 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-03-08T18:34:07.220 INFO:tasks.rook:['__class__', '__del__', '__delattr__', '__dir__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '_format_size', '_reconnect', '_runner', '_set_iface_and_cidr', '_sftp_get_file', '_sftp_get_size', '_sftp_open_file', '_sftp_put_file', 'chcon', 'chmod', 'connect', 'copy_file', 'ensure_online', 'get_file', 'get_tar', 'get_tar_stream', 'mkdtemp', 'mktemp', 'move_file', 'put_file', 'read_file', 'reconnect', 'remove', 'run', 'sh', 'sh_file', 'sudo_write_file', 'write_file'] 2022-03-08T18:34:07.238 DEBUG:teuthology.orchestra.remote:smithi002:rook/cluster/examples/kubernetes/ceph/operator.yaml is 22KB 2022-03-08T18:34:07.253 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:34:07.253 DEBUG:teuthology.orchestra.run.smithi002:> dd if=rook/cluster/examples/kubernetes/ceph/operator.yaml of=/dev/stdout 2022-03-08T18:34:07.262 INFO:tasks.rook:Patching operator to use image rook/ceph:v1.7.2 2022-03-08T18:34:07.302 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:34:07.303 DEBUG:teuthology.orchestra.run.smithi002:> dd of=operator.yaml 2022-03-08T18:34:07.310 INFO:tasks.rook:Deploying operator 2022-03-08T18:34:07.329 DEBUG:teuthology.orchestra.remote:smithi002:rook/cluster/examples/kubernetes/ceph/operator.yaml is 22KB 2022-03-08T18:34:07.366 DEBUG:teuthology.orchestra.remote:smithi002:rook/cluster/examples/kubernetes/ceph/operator.yaml is 22KB 2022-03-08T18:34:07.380 DEBUG:teuthology.orchestra.run.smithi002:> kubectl create -f rook/cluster/examples/kubernetes/ceph/crds.yaml -f rook/cluster/examples/kubernetes/ceph/common.yaml -f operator.yaml 2022-03-08T18:34:07.635 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/cephblockpools.ceph.rook.io created 2022-03-08T18:34:07.659 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/cephclients.ceph.rook.io created 2022-03-08T18:34:07.820 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/cephclusters.ceph.rook.io created 2022-03-08T18:34:07.838 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/cephfilesystemmirrors.ceph.rook.io created 2022-03-08T18:34:07.883 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/cephfilesystems.ceph.rook.io created 2022-03-08T18:34:07.986 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/cephnfses.ceph.rook.io created 2022-03-08T18:34:08.000 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/cephobjectrealms.ceph.rook.io created 2022-03-08T18:34:08.056 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/cephobjectstores.ceph.rook.io created 2022-03-08T18:34:08.078 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/cephobjectstoreusers.ceph.rook.io created 2022-03-08T18:34:08.167 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/cephobjectzonegroups.ceph.rook.io created 2022-03-08T18:34:08.347 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/cephobjectzones.ceph.rook.io created 2022-03-08T18:34:08.388 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/cephrbdmirrors.ceph.rook.io created 2022-03-08T18:34:08.413 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/objectbucketclaims.objectbucket.io created 2022-03-08T18:34:08.439 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/objectbuckets.objectbucket.io created 2022-03-08T18:34:08.535 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/volumereplicationclasses.replication.storage.openshift.io created 2022-03-08T18:34:08.550 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/volumereplications.replication.storage.openshift.io created 2022-03-08T18:34:08.576 INFO:teuthology.orchestra.run.smithi002.stdout:customresourcedefinition.apiextensions.k8s.io/volumes.rook.io created 2022-03-08T18:34:08.726 INFO:teuthology.orchestra.run.smithi002.stdout:namespace/rook-ceph created 2022-03-08T18:34:08.733 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-object-bucket created 2022-03-08T18:34:08.741 INFO:teuthology.orchestra.run.smithi002.stdout:serviceaccount/rook-ceph-admission-controller created 2022-03-08T18:34:08.767 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/rook-ceph-admission-controller-role created 2022-03-08T18:34:08.793 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-admission-controller-rolebinding created 2022-03-08T18:34:08.819 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/rook-ceph-cluster-mgmt created 2022-03-08T18:34:08.895 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/rook-ceph-system created 2022-03-08T18:34:08.899 INFO:teuthology.orchestra.run.smithi002.stdout:role.rbac.authorization.k8s.io/rook-ceph-system created 2022-03-08T18:34:08.908 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/rook-ceph-global created 2022-03-08T18:34:08.916 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/rook-ceph-mgr-cluster created 2022-03-08T18:34:08.925 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/rook-ceph-object-bucket created 2022-03-08T18:34:08.951 INFO:teuthology.orchestra.run.smithi002.stdout:serviceaccount/rook-ceph-system created 2022-03-08T18:34:08.958 INFO:teuthology.orchestra.run.smithi002.stdout:rolebinding.rbac.authorization.k8s.io/rook-ceph-system created 2022-03-08T18:34:08.966 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-system created 2022-03-08T18:34:08.993 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-global created 2022-03-08T18:34:09.061 INFO:teuthology.orchestra.run.smithi002.stdout:serviceaccount/rook-ceph-osd created 2022-03-08T18:34:09.064 INFO:teuthology.orchestra.run.smithi002.stdout:serviceaccount/rook-ceph-mgr created 2022-03-08T18:34:09.075 INFO:teuthology.orchestra.run.smithi002.stdout:serviceaccount/rook-ceph-cmd-reporter created 2022-03-08T18:34:09.102 INFO:teuthology.orchestra.run.smithi002.stdout:role.rbac.authorization.k8s.io/rook-ceph-osd created 2022-03-08T18:34:09.127 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/rook-ceph-osd created 2022-03-08T18:34:09.152 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/rook-ceph-mgr-system created 2022-03-08T18:34:09.158 INFO:teuthology.orchestra.run.smithi002.stdout:role.rbac.authorization.k8s.io/rook-ceph-mgr created 2022-03-08T18:34:09.227 INFO:teuthology.orchestra.run.smithi002.stdout:role.rbac.authorization.k8s.io/rook-ceph-cmd-reporter created 2022-03-08T18:34:09.230 INFO:teuthology.orchestra.run.smithi002.stdout:rolebinding.rbac.authorization.k8s.io/rook-ceph-cluster-mgmt created 2022-03-08T18:34:09.243 INFO:teuthology.orchestra.run.smithi002.stdout:rolebinding.rbac.authorization.k8s.io/rook-ceph-osd created 2022-03-08T18:34:09.249 INFO:teuthology.orchestra.run.smithi002.stdout:rolebinding.rbac.authorization.k8s.io/rook-ceph-mgr created 2022-03-08T18:34:09.258 INFO:teuthology.orchestra.run.smithi002.stdout:rolebinding.rbac.authorization.k8s.io/rook-ceph-mgr-system created 2022-03-08T18:34:09.266 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-mgr-cluster created 2022-03-08T18:34:09.274 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-osd created 2022-03-08T18:34:09.302 INFO:teuthology.orchestra.run.smithi002.stdout:rolebinding.rbac.authorization.k8s.io/rook-ceph-cmd-reporter created 2022-03-08T18:34:09.307 INFO:teuthology.orchestra.run.smithi002.stderr:Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ 2022-03-08T18:34:09.308 INFO:teuthology.orchestra.run.smithi002.stdout:podsecuritypolicy.policy/00-rook-privileged created 2022-03-08T18:34:09.316 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/psp:rook created 2022-03-08T18:34:09.324 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-system-psp created 2022-03-08T18:34:09.394 INFO:teuthology.orchestra.run.smithi002.stdout:rolebinding.rbac.authorization.k8s.io/rook-ceph-default-psp created 2022-03-08T18:34:09.397 INFO:teuthology.orchestra.run.smithi002.stdout:rolebinding.rbac.authorization.k8s.io/rook-ceph-osd-psp created 2022-03-08T18:34:09.408 INFO:teuthology.orchestra.run.smithi002.stdout:rolebinding.rbac.authorization.k8s.io/rook-ceph-mgr-psp created 2022-03-08T18:34:09.416 INFO:teuthology.orchestra.run.smithi002.stdout:rolebinding.rbac.authorization.k8s.io/rook-ceph-cmd-reporter-psp created 2022-03-08T18:34:09.424 INFO:teuthology.orchestra.run.smithi002.stdout:serviceaccount/rook-csi-cephfs-plugin-sa created 2022-03-08T18:34:09.433 INFO:teuthology.orchestra.run.smithi002.stdout:serviceaccount/rook-csi-cephfs-provisioner-sa created 2022-03-08T18:34:09.441 INFO:teuthology.orchestra.run.smithi002.stdout:role.rbac.authorization.k8s.io/cephfs-external-provisioner-cfg created 2022-03-08T18:34:09.486 INFO:teuthology.orchestra.run.smithi002.stdout:rolebinding.rbac.authorization.k8s.io/cephfs-csi-provisioner-role-cfg created 2022-03-08T18:34:09.552 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/cephfs-csi-nodeplugin created 2022-03-08T18:34:09.556 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/cephfs-external-provisioner-runner created 2022-03-08T18:34:09.586 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/rook-csi-cephfs-plugin-sa-psp created 2022-03-08T18:34:09.591 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/rook-csi-cephfs-provisioner-sa-psp created 2022-03-08T18:34:09.599 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/cephfs-csi-nodeplugin created 2022-03-08T18:34:09.628 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/cephfs-csi-provisioner-role created 2022-03-08T18:34:09.633 INFO:teuthology.orchestra.run.smithi002.stdout:serviceaccount/rook-csi-rbd-plugin-sa created 2022-03-08T18:34:09.641 INFO:teuthology.orchestra.run.smithi002.stdout:serviceaccount/rook-csi-rbd-provisioner-sa created 2022-03-08T18:34:09.650 INFO:teuthology.orchestra.run.smithi002.stdout:role.rbac.authorization.k8s.io/rbd-external-provisioner-cfg created 2022-03-08T18:34:09.737 INFO:teuthology.orchestra.run.smithi002.stdout:rolebinding.rbac.authorization.k8s.io/rbd-csi-provisioner-role-cfg created 2022-03-08T18:34:09.740 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/rbd-csi-nodeplugin created 2022-03-08T18:34:09.750 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrole.rbac.authorization.k8s.io/rbd-external-provisioner-runner created 2022-03-08T18:34:09.758 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/rook-csi-rbd-plugin-sa-psp created 2022-03-08T18:34:09.786 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/rook-csi-rbd-provisioner-sa-psp created 2022-03-08T18:34:09.791 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/rbd-csi-nodeplugin created 2022-03-08T18:34:09.799 INFO:teuthology.orchestra.run.smithi002.stdout:clusterrolebinding.rbac.authorization.k8s.io/rbd-csi-provisioner-role created 2022-03-08T18:34:09.808 INFO:teuthology.orchestra.run.smithi002.stdout:role.rbac.authorization.k8s.io/rook-ceph-purge-osd created 2022-03-08T18:34:09.816 INFO:teuthology.orchestra.run.smithi002.stdout:rolebinding.rbac.authorization.k8s.io/rook-ceph-purge-osd created 2022-03-08T18:34:09.824 INFO:teuthology.orchestra.run.smithi002.stdout:serviceaccount/rook-ceph-purge-osd created 2022-03-08T18:34:09.833 INFO:teuthology.orchestra.run.smithi002.stdout:configmap/rook-ceph-operator-config created 2022-03-08T18:34:09.920 INFO:teuthology.orchestra.run.smithi002.stdout:deployment.apps/rook-ceph-operator created 2022-03-08T18:34:09.927 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-operator 2022-03-08T18:34:10.099 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:34:10.099 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-operator-6b496c8959-6ltxg 0/1 Pending 0 1s 2022-03-08T18:34:20.103 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-operator 2022-03-08T18:34:20.174 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:34:20.174 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-operator-6b496c8959-6ltxg 0/1 Pending 0 11s 2022-03-08T18:34:30.177 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-operator 2022-03-08T18:34:30.392 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:34:30.392 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-operator-6b496c8959-6ltxg 0/1 Pending 0 21s 2022-03-08T18:34:40.397 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-operator 2022-03-08T18:34:40.481 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:34:40.482 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-operator-6b496c8959-6ltxg 0/1 ContainerCreating 0 31s 2022-03-08T18:34:50.486 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-operator 2022-03-08T18:34:50.558 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:34:50.559 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-operator-6b496c8959-6ltxg 0/1 ContainerCreating 0 41s 2022-03-08T18:35:00.562 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-operator 2022-03-08T18:35:00.623 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:35:00.623 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-operator-6b496c8959-6ltxg 0/1 ContainerCreating 0 51s 2022-03-08T18:35:10.626 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-operator 2022-03-08T18:35:10.695 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:35:10.695 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-operator-6b496c8959-6ltxg 0/1 ContainerCreating 0 61s 2022-03-08T18:35:20.698 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-operator 2022-03-08T18:35:20.764 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:35:20.765 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-operator-6b496c8959-6ltxg 1/1 Running 0 71s 2022-03-08T18:35:20.767 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph logs -f rook-ceph-operator-6b496c8959-6ltxg 2022-03-08T18:35:20.775 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:35:20.775 DEBUG:teuthology.orchestra.run.smithi002:> dd if=/scratch_devs of=/dev/stdout 2022-03-08T18:35:20.781 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:35:20.782 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/scratch_devs of=/dev/stdout 2022-03-08T18:35:20.790 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:35:20.790 DEBUG:teuthology.orchestra.run.smithi137:> dd if=/scratch_devs of=/dev/stdout 2022-03-08T18:35:20.799 INFO:tasks.rook: override: [mgr] debug mgr = 20 2022-03-08T18:35:20.800 INFO:tasks.rook: override: [mgr] debug ms = 1 2022-03-08T18:35:20.801 INFO:tasks.rook: override: [mon] debug mon = 20 2022-03-08T18:35:20.801 INFO:tasks.rook: override: [mon] debug ms = 1 2022-03-08T18:35:20.802 INFO:tasks.rook: override: [mon] debug paxos = 20 2022-03-08T18:35:20.802 INFO:tasks.rook: override: [osd] debug ms = 1 2022-03-08T18:35:20.803 INFO:tasks.rook: override: [osd] debug osd = 20 2022-03-08T18:35:20.804 INFO:tasks.rook:Config: b"[global]\n\nlog to file = true\n\nmon clock drift allowed = 1.000\n\n# replicate across OSDs, not hosts\nosd crush chooseleaf type = 0\n\n# enable some debugging\nauth debug = true\nms die on old message = true\nms die on bug = true\ndebug asserts on shutdown = true\n\n\n[osd]\n# debugging\nosd debug shutdown = true\nosd debug op order = true\nosd debug verify stray on activate = true\nosd debug pg log writeout = true\nosd debug verify cached snaps = true\nosd debug verify missing on start = true\nosd debug misdirected ops = true\nosd op queue = debug_random\nosd op queue cut off = debug_random\nosd shutdown pgref assert = true\nbdev debug aio = true\nosd sloppy crc = true\ndebug ms = 1\ndebug osd = 20\n\n\n[mon]\n# rotate auth tickets quickly to exercise renewal paths\nauth mon ticket ttl = 660# 11m\nauth service ticket ttl = 240# 4m\n\n# don't complain about global id reclaim\nmon_warn_on_insecure_global_id_reclaim = false\nmon_warn_on_insecure_global_id_reclaim_allowed = false\ndebug mon = 20\ndebug ms = 1\ndebug paxos = 20\n[mgr]\ndebug mgr = 20\ndebug ms = 1\n" 2022-03-08T18:35:20.809 DEBUG:teuthology.orchestra.run.smithi002:> kubectl create -f - 2022-03-08T18:35:20.868 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:15.793054 I | rookcmd: starting Rook v1.7.2 with arguments '/usr/local/bin/rook ceph operator' 2022-03-08T18:35:20.869 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:15.793159 I | rookcmd: flag values: --add_dir_header=false, --alsologtostderr=false, --csi-cephfs-plugin-template-path=/etc/ceph-csi/cephfs/csi-cephfsplugin.yaml, --csi-cephfs-provisioner-dep-template-path=/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner-dep.yaml, --csi-rbd-plugin-template-path=/etc/ceph-csi/rbd/csi-rbdplugin.yaml, --csi-rbd-provisioner-dep-template-path=/etc/ceph-csi/rbd/csi-rbdplugin-provisioner-dep.yaml, --enable-machine-disruption-budget=false, --help=false, --kubeconfig=, --log-flush-frequency=5s, --log-level=INFO, --log_backtrace_at=:0, --log_dir=, --log_file=, --log_file_max_size=1800, --logtostderr=true, --one_output=false, --operator-image=, --service-account=, --skip_headers=false, --skip_log_headers=false, --stderrthreshold=2, --v=0, --vmodule= 2022-03-08T18:35:20.869 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:15.793166 I | cephcmd: starting Rook-Ceph operator 2022-03-08T18:35:20.869 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:15.936380 I | cephcmd: base ceph version inside the rook operator image is "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)" 2022-03-08T18:35:20.869 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:15.979898 I | op-k8sutil: ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS="15" (configmap) 2022-03-08T18:35:20.870 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.035806 I | op-k8sutil: ROOK_ENABLE_DISCOVERY_DAEMON="false" (configmap) 2022-03-08T18:35:20.870 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.039326 I | operator: looking for secret "rook-ceph-admission-controller" 2022-03-08T18:35:20.870 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.041725 I | operator: secret "rook-ceph-admission-controller" not found. proceeding without the admission controller 2022-03-08T18:35:20.870 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.044903 I | op-k8sutil: ROOK_ENABLE_FLEX_DRIVER="false" (configmap) 2022-03-08T18:35:20.870 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.044945 I | operator: watching all namespaces for ceph cluster CRs 2022-03-08T18:35:20.871 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.045069 I | operator: setting up the controller-runtime manager 2022-03-08T18:35:20.871 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.047234 I | ceph-cluster-controller: ConfigMap "rook-ceph-operator-config" changes detected. Updating configurations 2022-03-08T18:35:20.871 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.050418 I | op-k8sutil: ROOK_LOG_LEVEL="INFO" (configmap) 2022-03-08T18:35:20.871 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.454629 I | ceph-cluster-controller: successfully started 2022-03-08T18:35:20.872 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.454718 I | ceph-cluster-controller: enabling hotplug orchestration 2022-03-08T18:35:20.872 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.454753 I | ceph-crashcollector-controller: successfully started 2022-03-08T18:35:20.872 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.454841 I | ceph-block-pool-controller: successfully started 2022-03-08T18:35:20.872 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.454923 I | ceph-object-store-user-controller: successfully started 2022-03-08T18:35:20.872 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.455005 I | ceph-object-realm-controller: successfully started 2022-03-08T18:35:20.873 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.455074 I | ceph-object-zonegroup-controller: successfully started 2022-03-08T18:35:20.873 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.455145 I | ceph-object-zone-controller: successfully started 2022-03-08T18:35:20.873 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.455336 I | ceph-object-controller: successfully started 2022-03-08T18:35:20.873 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.455453 I | ceph-file-controller: successfully started 2022-03-08T18:35:20.874 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.455557 I | ceph-nfs-controller: successfully started 2022-03-08T18:35:20.874 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.455671 I | ceph-rbd-mirror-controller: successfully started 2022-03-08T18:35:20.874 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.455766 I | ceph-client-controller: successfully started 2022-03-08T18:35:20.874 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.455851 I | ceph-filesystem-mirror-controller: successfully started 2022-03-08T18:35:20.874 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:16.457315 I | operator: starting the controller-runtime manager 2022-03-08T18:35:21.840 INFO:teuthology.orchestra.run.smithi002.stdout:configmap/rook-config-override created 2022-03-08T18:35:21.844 INFO:tasks.rook:Cluster: apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: name: rook-ceph namespace: rook-ceph spec: cephVersion: allowUnsupported: true image: quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae dataDirHostPath: /var/lib/rook mgr: count: 1 modules: - enabled: true name: rook mon: allowMultiplePerNode: true count: 3 skipUpgradeChecks: true 2022-03-08T18:35:21.844 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:35:21.844 DEBUG:teuthology.orchestra.run.smithi002:> dd of=cluster.yaml 2022-03-08T18:35:21.851 DEBUG:teuthology.orchestra.run.smithi002:> kubectl create -f cluster.yaml 2022-03-08T18:35:24.268 INFO:teuthology.orchestra.run.smithi002.stdout:cephcluster.ceph.rook.io/rook-ceph created 2022-03-08T18:35:24.274 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.274258 I | clusterdisruption-controller: create event from ceph cluster CR 2022-03-08T18:35:24.275 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.274317 I | ceph-spec: adding finalizer "cephcluster.ceph.rook.io" on "rook-ceph" 2022-03-08T18:35:24.286 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.285476 I | ceph-cluster-controller: reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:35:24.287 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.285611 I | ceph-cluster-controller: CR has changed for "rook-ceph". diff=  v1.ClusterSpec{ 2022-03-08T18:35:24.287 INFO:tasks.rook.operator.smithi002.stdout:   ... // 3 identical fields 2022-03-08T18:35:24.287 INFO:tasks.rook.operator.smithi002.stdout:   Labels: nil, 2022-03-08T18:35:24.287 INFO:tasks.rook.operator.smithi002.stdout:   Placement: nil, 2022-03-08T18:35:24.288 INFO:tasks.rook.operator.smithi002.stdout:   Network: v1.NetworkSpec{ 2022-03-08T18:35:24.288 INFO:tasks.rook.operator.smithi002.stdout:   Provider: "", 2022-03-08T18:35:24.288 INFO:tasks.rook.operator.smithi002.stdout:   Selectors: nil, 2022-03-08T18:35:24.288 INFO:tasks.rook.operator.smithi002.stdout:   HostNetwork: false, 2022-03-08T18:35:24.288 INFO:tasks.rook.operator.smithi002.stdout:-  IPFamily: "", 2022-03-08T18:35:24.289 INFO:tasks.rook.operator.smithi002.stdout:+  IPFamily: "IPv4", 2022-03-08T18:35:24.289 INFO:tasks.rook.operator.smithi002.stdout:   DualStack: false, 2022-03-08T18:35:24.289 INFO:tasks.rook.operator.smithi002.stdout:   }, 2022-03-08T18:35:24.289 INFO:tasks.rook.operator.smithi002.stdout:   Resources: nil, 2022-03-08T18:35:24.290 INFO:tasks.rook.operator.smithi002.stdout:   PriorityClassNames: nil, 2022-03-08T18:35:24.290 INFO:tasks.rook.operator.smithi002.stdout:   ... // 16 identical fields 2022-03-08T18:35:24.290 INFO:tasks.rook.operator.smithi002.stdout:  } 2022-03-08T18:35:24.292 DEBUG:teuthology.orchestra.remote:smithi002:rook/cluster/examples/kubernetes/ceph/operator.yaml is 22KB 2022-03-08T18:35:24.308 DEBUG:teuthology.orchestra.run.smithi002:> kubectl create -f rook/cluster/examples/kubernetes/ceph/toolbox.yaml 2022-03-08T18:35:24.346 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.346439 I | op-k8sutil: ROOK_ENABLE_FLEX_DRIVER="false" (configmap) 2022-03-08T18:35:24.376 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.376569 I | ceph-csi: successfully created csi config map "rook-ceph-csi-config" 2022-03-08T18:35:24.379 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.379100 I | ceph-cluster-controller: clusterInfo not yet found, must be a new cluster 2022-03-08T18:35:24.379 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.379342 I | op-k8sutil: ROOK_CSI_ENABLE_RBD="true" (configmap) 2022-03-08T18:35:24.382 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.381440 I | op-k8sutil: ROOK_CSI_ENABLE_CEPHFS="true" (configmap) 2022-03-08T18:35:24.388 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.388680 I | op-k8sutil: ROOK_CSI_ALLOW_UNSUPPORTED_VERSION="false" (configmap) 2022-03-08T18:35:24.391 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.391112 I | op-k8sutil: ROOK_CSI_ENABLE_GRPC_METRICS="false" (configmap) 2022-03-08T18:35:24.394 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.393707 I | op-k8sutil: CSI_ENABLE_HOST_NETWORK="true" (default) 2022-03-08T18:35:24.396 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.396666 I | ceph-cluster-controller: detecting the ceph image version for image quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae... 2022-03-08T18:35:24.446 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.445943 I | op-k8sutil: ROOK_CSI_CEPH_IMAGE="quay.io/cephcsi/cephcsi:v3.4.0" (default) 2022-03-08T18:35:24.621 INFO:teuthology.orchestra.run.smithi002.stdout:deployment.apps/rook-ceph-tools created 2022-03-08T18:35:24.625 INFO:tasks.rook:Waiting for tools container to start 2022-03-08T18:35:24.626 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:35:24.785 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:24.784572 I | op-k8sutil: ROOK_CSI_REGISTRAR_IMAGE="k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0" (default) 2022-03-08T18:35:24.786 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:35:24.787 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 0s 2022-03-08T18:35:25.091 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:25.091061 I | op-k8sutil: ROOK_CSI_PROVISIONER_IMAGE="k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2" (default) 2022-03-08T18:35:25.291 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:25.290965 I | op-k8sutil: ROOK_CSI_ATTACHER_IMAGE="k8s.gcr.io/sig-storage/csi-attacher:v3.2.1" (default) 2022-03-08T18:35:25.492 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:25.491180 I | op-k8sutil: ROOK_CSI_SNAPSHOTTER_IMAGE="k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1" (default) 2022-03-08T18:35:25.689 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:25.688911 I | op-k8sutil: ROOK_CSI_KUBELET_DIR_PATH="/var/lib/kubelet" (default) 2022-03-08T18:35:25.892 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:25.890865 I | op-k8sutil: CSI_VOLUME_REPLICATION_IMAGE="quay.io/csiaddons/volumereplication-operator:v0.1.0" (default) 2022-03-08T18:35:26.090 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:26.089510 I | op-k8sutil: ROOK_CSI_CEPHFS_POD_LABELS="" (default) 2022-03-08T18:35:26.289 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:26.288571 I | op-k8sutil: ROOK_CSI_RBD_POD_LABELS="" (default) 2022-03-08T18:35:26.290 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:26.288605 I | ceph-csi: detecting the ceph csi image version for image "quay.io/cephcsi/cephcsi:v3.4.0" 2022-03-08T18:35:26.490 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:35:26.489318 I | op-k8sutil: CSI_PROVISIONER_TOLERATIONS="" (default) 2022-03-08T18:35:29.790 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:35:29.857 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:35:29.858 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 5s 2022-03-08T18:35:34.860 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:35:34.942 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:35:34.942 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 10s 2022-03-08T18:35:39.946 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:35:40.012 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:35:40.013 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 16s 2022-03-08T18:35:45.015 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:35:45.079 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:35:45.080 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 21s 2022-03-08T18:35:50.083 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:35:50.149 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:35:50.150 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 26s 2022-03-08T18:35:55.152 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:35:55.218 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:35:55.218 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 31s 2022-03-08T18:36:00.221 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:36:00.295 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:36:00.295 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 36s 2022-03-08T18:36:05.299 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:36:05.371 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:36:05.372 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 41s 2022-03-08T18:36:10.374 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:36:10.443 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:36:10.443 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 46s 2022-03-08T18:36:15.446 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:36:15.524 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:36:15.525 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 51s 2022-03-08T18:36:20.528 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:36:20.601 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:36:20.601 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 56s 2022-03-08T18:36:25.604 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:36:25.680 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:36:25.681 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 61s 2022-03-08T18:36:27.980 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:27.979816 I | ceph-csi: Detected ceph CSI image version: "v3.4.0" 2022-03-08T18:36:27.990 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:27.987007 I | op-k8sutil: CSI_FORCE_CEPHFS_KERNEL_CLIENT="true" (configmap) 2022-03-08T18:36:27.990 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:27.989449 I | op-k8sutil: CSI_CEPHFS_GRPC_METRICS_PORT="9091" (default) 2022-03-08T18:36:28.045 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:28.044241 I | op-k8sutil: CSI_CEPHFS_LIVENESS_METRICS_PORT="9081" (default) 2022-03-08T18:36:28.047 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:28.046785 I | op-k8sutil: CSI_RBD_GRPC_METRICS_PORT="9090" (default) 2022-03-08T18:36:28.055 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:28.053588 I | op-k8sutil: CSI_RBD_LIVENESS_METRICS_PORT="9080" (default) 2022-03-08T18:36:28.062 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:28.062095 I | op-k8sutil: CSI_PLUGIN_PRIORITY_CLASSNAME="" (default) 2022-03-08T18:36:28.111 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:28.111151 I | op-k8sutil: CSI_PROVISIONER_PRIORITY_CLASSNAME="" (default) 2022-03-08T18:36:28.200 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:28.199113 I | ceph-cluster-controller: detected ceph image version: "17.0.0-10961 quincy" 2022-03-08T18:36:28.200 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:28.199137 I | ceph-cluster-controller: validating ceph version from provided image 2022-03-08T18:36:28.200 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:28.199149 W | ceph-cluster-controller: unsupported ceph version detected: "17.0.0-10961 quincy", pursuing 2022-03-08T18:36:28.355 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:28.354844 I | op-k8sutil: CSI_ENABLE_OMAP_GENERATOR="false" (default) 2022-03-08T18:36:28.556 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:28.555363 I | ceph-cluster-controller: cluster "rook-ceph": version "17.0.0-10961 quincy" detected for image "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae" 2022-03-08T18:36:28.756 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:28.755726 I | op-k8sutil: CSI_ENABLE_RBD_SNAPSHOTTER="true" (configmap) 2022-03-08T18:36:29.055 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:29.054433 I | op-mon: start running mons 2022-03-08T18:36:29.206 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:29.205753 I | op-k8sutil: CSI_ENABLE_CEPHFS_SNAPSHOTTER="true" (configmap) 2022-03-08T18:36:29.423 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:29.422024 I | op-mon: creating mon secrets for a new cluster 2022-03-08T18:36:29.555 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:29.554807 I | op-k8sutil: CSI_ENABLE_VOLUME_REPLICATION="false" (configmap) 2022-03-08T18:36:29.956 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:29.955590 I | op-k8sutil: CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY="RollingUpdate" (default) 2022-03-08T18:36:30.356 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:30.355882 I | op-k8sutil: CSI_RBD_PLUGIN_UPDATE_STRATEGY="RollingUpdate" (default) 2022-03-08T18:36:30.357 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:30.355912 I | ceph-csi: Kubernetes version is 1.23 2022-03-08T18:36:30.555 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:30.554518 I | op-mon: existing maxMonID not found or failed to load. configmaps "rook-ceph-mon-endpoints" not found 2022-03-08T18:36:30.684 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:36:30.756 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:30.755046 I | op-k8sutil: ROOK_CSI_RESIZER_IMAGE="k8s.gcr.io/sig-storage/csi-resizer:v1.2.0" (default) 2022-03-08T18:36:30.760 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:36:30.760 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 66s 2022-03-08T18:36:30.958 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:30.957330 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":[]}] data: mapping:{"node":{}} maxMonId:-1] 2022-03-08T18:36:31.155 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:31.154701 I | op-k8sutil: CSI_LOG_LEVEL="" (default) 2022-03-08T18:36:31.956 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:31.955225 I | op-k8sutil: CSI_PROVISIONER_TOLERATIONS="" (default) 2022-03-08T18:36:32.355 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:32.354544 I | op-k8sutil: CSI_PROVISIONER_NODE_AFFINITY="" (default) 2022-03-08T18:36:32.556 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:32.555499 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:36:32.556 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:32.555696 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:36:32.755 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:32.755071 I | op-k8sutil: CSI_PLUGIN_TOLERATIONS="" (default) 2022-03-08T18:36:33.157 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:33.155491 I | op-k8sutil: CSI_PLUGIN_NODE_AFFINITY="" (default) 2022-03-08T18:36:33.555 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:33.554400 I | op-k8sutil: CSI_RBD_PLUGIN_TOLERATIONS="" (default) 2022-03-08T18:36:33.955 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:33.954908 I | op-k8sutil: CSI_RBD_PLUGIN_NODE_AFFINITY="" (default) 2022-03-08T18:36:34.356 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:34.356298 I | op-k8sutil: CSI_RBD_PLUGIN_RESOURCE="" (default) 2022-03-08T18:36:34.780 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:34.779834 I | op-k8sutil: CSI_RBD_PROVISIONER_TOLERATIONS="" (default) 2022-03-08T18:36:34.955 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:34.955004 I | op-mon: targeting the mon count 3 2022-03-08T18:36:35.132 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:35.131696 I | ceph-cluster-controller: Cluster update requested. CANCELLING CURRENT ORCHESTRATION 2022-03-08T18:36:35.132 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:35.131806 I | ceph-cluster-controller: reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:36:35.309 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:35.308810 I | op-k8sutil: CSI_RBD_PROVISIONER_NODE_AFFINITY="" (default) 2022-03-08T18:36:35.556 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:35.555674 I | op-k8sutil: CSI_RBD_PROVISIONER_RESOURCE="" (default) 2022-03-08T18:36:35.571 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:35.571448 I | ceph-csi: successfully started CSI Ceph RBD driver 2022-03-08T18:36:35.755 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:35.754585 I | op-mon: parsing mon endpoints: 2022-03-08T18:36:35.755 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:35.754615 W | op-mon: ignoring invalid monitor 2022-03-08T18:36:35.763 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:36:35.807 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:35.805621 I | ceph-cluster-controller: detecting the ceph image version for image quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae... 2022-03-08T18:36:35.967 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:36:35.967 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 71s 2022-03-08T18:36:36.382 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:36.381396 I | op-k8sutil: CSI_CEPHFS_PLUGIN_TOLERATIONS="" (default) 2022-03-08T18:36:36.756 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:36.755552 I | op-k8sutil: CSI_CEPHFS_PLUGIN_NODE_AFFINITY="" (default) 2022-03-08T18:36:36.956 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:36.955480 I | op-k8sutil: CSI_CEPHFS_PLUGIN_RESOURCE="" (default) 2022-03-08T18:36:37.208 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:37.207760 I | op-k8sutil: CSI_CEPHFS_PROVISIONER_TOLERATIONS="" (default) 2022-03-08T18:36:37.356 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:37.355595 I | op-k8sutil: CSI_CEPHFS_PROVISIONER_NODE_AFFINITY="" (default) 2022-03-08T18:36:37.555 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:37.554288 I | op-k8sutil: CSI_CEPHFS_PROVISIONER_RESOURCE="" (default) 2022-03-08T18:36:37.569 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:37.567285 I | ceph-csi: successfully started CSI CephFS driver 2022-03-08T18:36:38.558 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:38.553951 I | op-k8sutil: CSI_RBD_FSGROUPPOLICY="ReadWriteOnceWithFSType" (configmap) 2022-03-08T18:36:38.814 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:38.813406 I | ceph-csi: CSIDriver object created for driver "rook-ceph.rbd.csi.ceph.com" 2022-03-08T18:36:38.856 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:38.855989 I | op-k8sutil: CSI_CEPHFS_FSGROUPPOLICY="None" (configmap) 2022-03-08T18:36:38.949 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:38.948685 I | ceph-csi: CSIDriver object created for driver "rook-ceph.cephfs.csi.ceph.com" 2022-03-08T18:36:40.969 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:36:41.273 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:36:41.274 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 77s 2022-03-08T18:36:46.278 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:36:46.345 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:36:46.345 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 82s 2022-03-08T18:36:49.527 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:49.526795 I | ceph-cluster-controller: detected ceph image version: "17.0.0-10961 quincy" 2022-03-08T18:36:49.527 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:49.526822 I | ceph-cluster-controller: validating ceph version from provided image 2022-03-08T18:36:49.528 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:49.526841 W | ceph-cluster-controller: unsupported ceph version detected: "17.0.0-10961 quincy", pursuing 2022-03-08T18:36:49.581 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:49.580906 I | op-mon: parsing mon endpoints: 2022-03-08T18:36:49.581 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:49.580926 W | op-mon: ignoring invalid monitor 2022-03-08T18:36:49.634 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:49.633994 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:36:49.648 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:49.648291 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:36:49.776 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:49.775914 E | ceph-cluster-controller: failed to get ceph daemons versions, this typically happens during the first cluster initialization. failed to run 'ceph versions'. . unable to get monitor info from DNS SRV with service name: ceph-mon 2022-03-08T18:36:49.777 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T18:36:49.768+0000 7fb5cbfff700 -1 failed for service _ceph-mon._tcp 2022-03-08T18:36:49.777 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T18:36:49.768+0000 7fb5cbfff700 -1 monclient: get_monmap_and_config cannot identify monitors to contact 2022-03-08T18:36:49.777 INFO:tasks.rook.operator.smithi002.stdout:[errno 2] RADOS object not found (error connecting to the cluster): exit status 1 2022-03-08T18:36:49.778 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:49.775961 I | ceph-cluster-controller: cluster "rook-ceph": version "17.0.0-10961 quincy" detected for image "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae" 2022-03-08T18:36:49.856 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:49.856390 I | op-mon: start running mons 2022-03-08T18:36:49.887 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:49.887016 I | op-mon: parsing mon endpoints: 2022-03-08T18:36:49.887 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:49.887043 W | op-mon: ignoring invalid monitor 2022-03-08T18:36:49.900 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:49.900726 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":[]}] data: mapping:{"node":{}} maxMonId:-1] 2022-03-08T18:36:50.274 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:50.266530 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:36:50.274 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:50.274483 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:36:51.348 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:36:51.420 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:36:51.421 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 87s 2022-03-08T18:36:51.466 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:51.465220 I | op-mon: targeting the mon count 3 2022-03-08T18:36:51.471 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:51.471853 I | op-mon: sched-mon: created canary deployment rook-ceph-mon-a-canary 2022-03-08T18:36:51.491 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:51.489237 I | op-mon: sched-mon: created canary deployment rook-ceph-mon-b-canary 2022-03-08T18:36:51.582 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:51.581857 I | op-mon: sched-mon: created canary deployment rook-ceph-mon-c-canary 2022-03-08T18:36:52.360 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:52.360076 I | op-mon: sched-mon: canary monitor deployment rook-ceph-mon-a-canary scheduled to smithi137 2022-03-08T18:36:52.360 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:52.360099 I | op-mon: assignmon: mon a assigned to node smithi137 2022-03-08T18:36:52.734 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:52.733178 I | op-mon: sched-mon: canary monitor deployment rook-ceph-mon-b-canary scheduled to smithi066 2022-03-08T18:36:52.734 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:52.733201 I | op-mon: assignmon: mon b assigned to node smithi066 2022-03-08T18:36:52.735 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:52.733914 I | op-mon: sched-mon: canary monitor deployment rook-ceph-mon-c-canary scheduled to smithi002 2022-03-08T18:36:52.735 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:52.733937 I | op-mon: assignmon: mon c assigned to node smithi002 2022-03-08T18:36:52.739 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:52.739314 I | op-mon: cleaning up canary monitor deployment "rook-ceph-mon-a-canary" 2022-03-08T18:36:53.172 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:53.172805 I | op-mon: cleaning up canary monitor deployment "rook-ceph-mon-b-canary" 2022-03-08T18:36:54.116 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:54.113503 I | op-mon: cleaning up canary monitor deployment "rook-ceph-mon-c-canary" 2022-03-08T18:36:54.400 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:54.398959 I | op-mon: creating mon a 2022-03-08T18:36:55.702 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:55.701298 I | op-mon: mon "a" endpoint is [v2:10.101.136.169:3300,v1:10.101.136.169:6789] 2022-03-08T18:36:56.423 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:36:57.002 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:36:57.003 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 93s 2022-03-08T18:36:57.664 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:57.662041 I | op-mon: monitor endpoints changed, updating the bootstrap peer token 2022-03-08T18:36:57.664 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:57.662168 I | op-mon: monitor endpoints changed, updating the bootstrap peer token 2022-03-08T18:36:57.664 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:57.662340 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789"]}] data:a=10.101.136.169:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:-1] 2022-03-08T18:36:58.030 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:58.030223 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:36:58.033 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:58.030513 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:36:58.052 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:58.052394 I | op-mon: 0 of 1 expected mons are ready. creating or updating deployments without checking quorum in attempt to achieve a healthy mon cluster 2022-03-08T18:36:58.367 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:58.366128 I | op-mon: updating maxMonID from -1 to 0 after committing mon "a" 2022-03-08T18:36:58.553 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:58.553441 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789"]}] data:a=10.101.136.169:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:0] 2022-03-08T18:36:58.554 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:58.553473 I | op-mon: waiting for mon quorum with [a] 2022-03-08T18:36:58.559 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:58.560025 I | op-mon: mon a is not yet running 2022-03-08T18:36:58.560 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:36:58.560046 I | op-mon: mons running: [] 2022-03-08T18:37:02.005 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:37:02.107 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:37:02.107 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 98s 2022-03-08T18:37:07.110 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:37:07.179 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:37:07.180 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 103s 2022-03-08T18:37:12.183 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:37:12.250 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:37:12.251 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 108s 2022-03-08T18:37:17.253 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:37:17.322 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:37:17.322 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 113s 2022-03-08T18:37:18.679 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:18.678750 I | op-mon: mon a is not yet running 2022-03-08T18:37:18.679 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:18.678773 I | op-mon: mons running: [] 2022-03-08T18:37:22.325 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:37:22.398 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:37:22.398 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 118s 2022-03-08T18:37:27.400 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:37:27.477 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:37:27.477 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 2m3s 2022-03-08T18:37:32.480 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:37:32.926 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:37:32.926 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 2m8s 2022-03-08T18:37:37.929 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:37:38.369 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:37:38.370 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 2m14s 2022-03-08T18:37:38.919 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:38.919447 I | op-mon: mons running: [a] 2022-03-08T18:37:39.188 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:39.188614 I | op-mon: Monitors in quorum: [a] 2022-03-08T18:37:39.188 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:39.188643 I | op-mon: mons created: 1 2022-03-08T18:37:39.443 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:39.444369 I | op-mon: waiting for mon quorum with [a] 2022-03-08T18:37:39.453 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:39.454003 I | op-mon: mons running: [a] 2022-03-08T18:37:39.679 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:39.679331 I | op-mon: Monitors in quorum: [a] 2022-03-08T18:37:39.679 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:39.679374 I | op-config: setting "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:37:39.951 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:39.951349 I | op-config: successfully set "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:37:39.951 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:39.951386 I | op-config: setting "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:37:40.235 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:40.235287 I | op-config: successfully set "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:37:40.239 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:40.235319 I | op-config: setting "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:37:40.537 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:40.537454 I | op-config: successfully set "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:37:40.537 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:40.537489 I | op-config: setting "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:37:40.820 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:40.819988 I | op-config: successfully set "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:37:40.820 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:40.820020 I | op-config: setting "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:37:41.105 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:41.105371 I | op-config: successfully set "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:37:41.105 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:41.105410 I | op-config: deleting "log file" option from the mon configuration database 2022-03-08T18:37:41.330 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:41.330532 I | op-config: successfully deleted "log file" option from the mon configuration database 2022-03-08T18:37:41.330 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:41.330570 I | op-mon: creating mon b 2022-03-08T18:37:43.162 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:43.162932 I | op-mon: mon "a" endpoint is [v2:10.101.136.169:3300,v1:10.101.136.169:6789] 2022-03-08T18:37:43.372 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:37:43.805 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:43.803560 I | op-mon: mon "b" endpoint is [v2:10.110.64.66:3300,v1:10.110.64.66:6789] 2022-03-08T18:37:43.806 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:37:43.807 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 0/1 ContainerCreating 0 2m19s 2022-03-08T18:37:44.791 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:44.792087 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:0] 2022-03-08T18:37:45.691 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:45.691891 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:37:45.695 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:45.692265 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:37:46.427 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:46.427614 I | op-mon: deployment for mon rook-ceph-mon-a already exists. updating if needed 2022-03-08T18:37:46.437 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:46.438150 I | op-k8sutil: deployment "rook-ceph-mon-a" did not change, nothing to update 2022-03-08T18:37:46.437 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:46.438181 I | op-mon: waiting for mon quorum with [a b] 2022-03-08T18:37:46.681 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:46.681572 I | op-mon: mon b is not yet running 2022-03-08T18:37:46.681 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:46.681620 I | op-mon: mons running: [a] 2022-03-08T18:37:46.916 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:46.917677 I | op-mon: Monitors in quorum: [a] 2022-03-08T18:37:47.038 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:46.988275 I | op-mon: updating maxMonID from 0 to 1 after committing mon "b" 2022-03-08T18:37:47.248 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:47.249145 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:1] 2022-03-08T18:37:47.249 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:47.249180 I | op-mon: waiting for mon quorum with [a b] 2022-03-08T18:37:47.258 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:47.259176 I | op-mon: mon b is not yet running 2022-03-08T18:37:47.258 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:47.259221 I | op-mon: mons running: [a] 2022-03-08T18:37:47.481 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:47.482037 I | op-mon: Monitors in quorum: [a] 2022-03-08T18:37:47.481 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:47.482058 I | op-mon: mons created: 2 2022-03-08T18:37:47.710 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:47.711210 I | op-mon: waiting for mon quorum with [a b] 2022-03-08T18:37:48.170 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:48.171026 I | op-mon: mon b is not yet running 2022-03-08T18:37:48.170 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:48.171064 I | op-mon: mons running: [a] 2022-03-08T18:37:48.810 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph get pods -l app=rook-ceph-tools 2022-03-08T18:37:49.047 INFO:teuthology.orchestra.run.smithi002.stdout:NAME READY STATUS RESTARTS AGE 2022-03-08T18:37:49.047 INFO:teuthology.orchestra.run.smithi002.stdout:rook-ceph-tools-7677f7f4b9-6d9st 1/1 Running 0 2m25s 2022-03-08T18:37:49.049 INFO:tasks.rook:Waiting for mgr/rook orchestrator to be available 2022-03-08T18:37:49.050 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch status -f json 2022-03-08T18:37:49.791 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:37:49.791 INFO:teuthology.orchestra.run.smithi002.stderr:no valid command found; 10 closest matches: 2022-03-08T18:37:49.792 INFO:teuthology.orchestra.run.smithi002.stderr:pg stat 2022-03-08T18:37:49.792 INFO:teuthology.orchestra.run.smithi002.stderr:pg getmap 2022-03-08T18:37:49.792 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump [all|summary|sum|delta|pools|osds|pgs|pgs_brief...] 2022-03-08T18:37:49.793 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_json [all|summary|sum|pools|osds|pgs...] 2022-03-08T18:37:49.793 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_pools_json 2022-03-08T18:37:49.793 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-pool [...] 2022-03-08T18:37:49.794 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-primary [] [...] 2022-03-08T18:37:49.794 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-osd [] [...] 2022-03-08T18:37:49.794 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls [] [...] 2022-03-08T18:37:49.794 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_stuck [inactive|unclean|stale|undersized|degraded...] [] 2022-03-08T18:37:49.795 INFO:teuthology.orchestra.run.smithi002.stderr:Error EINVAL: invalid command 2022-03-08T18:37:49.795 INFO:teuthology.orchestra.run.smithi002.stderr:command terminated with exit code 22 2022-03-08T18:37:49.797 DEBUG:teuthology.orchestra.run:got remote process result: 22 2022-03-08T18:37:53.182 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:53.182981 I | op-mon: mon b is not yet running 2022-03-08T18:37:53.182 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:53.183013 I | op-mon: mons running: [a] 2022-03-08T18:37:58.201 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:37:58.202720 I | op-mon: mons running: [a b] 2022-03-08T18:37:59.800 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch status -f json 2022-03-08T18:38:00.972 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:38:00.973 INFO:teuthology.orchestra.run.smithi002.stderr:no valid command found; 10 closest matches: 2022-03-08T18:38:00.973 INFO:teuthology.orchestra.run.smithi002.stderr:pg stat 2022-03-08T18:38:00.973 INFO:teuthology.orchestra.run.smithi002.stderr:pg getmap 2022-03-08T18:38:00.974 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump [all|summary|sum|delta|pools|osds|pgs|pgs_brief...] 2022-03-08T18:38:00.974 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_json [all|summary|sum|pools|osds|pgs...] 2022-03-08T18:38:00.974 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_pools_json 2022-03-08T18:38:00.974 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-pool [...] 2022-03-08T18:38:00.975 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-primary [] [...] 2022-03-08T18:38:00.975 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-osd [] [...] 2022-03-08T18:38:00.975 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls [] [...] 2022-03-08T18:38:00.976 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_stuck [inactive|unclean|stale|undersized|degraded...] [] 2022-03-08T18:38:00.976 INFO:teuthology.orchestra.run.smithi002.stderr:Error EINVAL: invalid command 2022-03-08T18:38:00.978 INFO:teuthology.orchestra.run.smithi002.stderr:command terminated with exit code 22 2022-03-08T18:38:00.979 DEBUG:teuthology.orchestra.run:got remote process result: 22 2022-03-08T18:38:01.424 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:01.424987 I | op-mon: Monitors in quorum: [a b] 2022-03-08T18:38:01.424 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:01.425029 I | op-config: setting "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:38:01.649 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:01.650205 I | op-config: successfully set "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:38:01.649 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:01.650236 I | op-config: setting "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:38:01.873 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:01.873648 I | op-config: successfully set "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:38:01.873 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:01.873680 I | op-config: setting "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:38:02.084 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:02.085050 I | op-config: successfully set "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:38:02.084 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:02.085084 I | op-config: setting "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:38:02.306 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:02.306979 I | op-config: successfully set "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:38:02.306 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:02.307009 I | op-config: setting "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:38:02.530 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:02.530731 I | op-config: successfully set "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:38:02.530 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:02.530760 I | op-config: deleting "log file" option from the mon configuration database 2022-03-08T18:38:02.745 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:02.746112 I | op-config: successfully deleted "log file" option from the mon configuration database 2022-03-08T18:38:02.745 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:02.746143 I | op-mon: creating mon c 2022-03-08T18:38:02.916 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:02.917226 I | op-mon: mon "a" endpoint is [v2:10.101.136.169:3300,v1:10.101.136.169:6789] 2022-03-08T18:38:03.085 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:03.085985 I | op-mon: mon "b" endpoint is [v2:10.110.64.66:3300,v1:10.110.64.66:6789] 2022-03-08T18:38:03.209 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:03.210433 I | op-mon: mon "c" endpoint is [v2:10.109.10.123:3300,v1:10.109.10.123:6789] 2022-03-08T18:38:03.262 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:03.263770 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:1] 2022-03-08T18:38:04.091 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:04.092343 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:38:04.095 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:04.092669 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:38:04.416 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:04.418017 I | op-mon: 2 of 3 expected mon deployments exist. creating new deployment(s). 2022-03-08T18:38:04.420 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:04.422080 I | op-mon: deployment for mon rook-ceph-mon-a already exists. updating if needed 2022-03-08T18:38:04.430 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:04.429109 I | op-k8sutil: deployment "rook-ceph-mon-a" did not change, nothing to update 2022-03-08T18:38:04.494 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:04.495874 I | op-mon: deployment for mon rook-ceph-mon-b already exists. updating if needed 2022-03-08T18:38:04.595 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:04.596453 I | op-k8sutil: deployment "rook-ceph-mon-b" did not change, nothing to update 2022-03-08T18:38:04.643 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:04.644662 I | op-mon: updating maxMonID from 1 to 2 after committing mon "c" 2022-03-08T18:38:05.327 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:05.328378 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:38:05.328 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:05.328418 I | op-mon: waiting for mon quorum with [a b c] 2022-03-08T18:38:05.752 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:05.753267 I | op-mon: mon c is not yet running 2022-03-08T18:38:05.752 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:05.753307 I | op-mon: mons running: [a b] 2022-03-08T18:38:05.971 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:05.971934 I | op-mon: Monitors in quorum: [a b] 2022-03-08T18:38:05.971 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:05.971967 I | op-mon: mons created: 3 2022-03-08T18:38:06.185 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:06.186316 I | op-mon: waiting for mon quorum with [a b c] 2022-03-08T18:38:06.516 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:06.517205 I | op-mon: mon c is not yet running 2022-03-08T18:38:06.516 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:06.517239 I | op-mon: mons running: [a b] 2022-03-08T18:38:10.981 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch status -f json 2022-03-08T18:38:11.464 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:38:11.465 INFO:teuthology.orchestra.run.smithi002.stderr:no valid command found; 10 closest matches: 2022-03-08T18:38:11.465 INFO:teuthology.orchestra.run.smithi002.stderr:pg stat 2022-03-08T18:38:11.465 INFO:teuthology.orchestra.run.smithi002.stderr:pg getmap 2022-03-08T18:38:11.465 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump [all|summary|sum|delta|pools|osds|pgs|pgs_brief...] 2022-03-08T18:38:11.465 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_json [all|summary|sum|pools|osds|pgs...] 2022-03-08T18:38:11.466 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_pools_json 2022-03-08T18:38:11.466 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-pool [...] 2022-03-08T18:38:11.466 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-primary [] [...] 2022-03-08T18:38:11.467 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-osd [] [...] 2022-03-08T18:38:11.467 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls [] [...] 2022-03-08T18:38:11.467 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_stuck [inactive|unclean|stale|undersized|degraded...] [] 2022-03-08T18:38:11.467 INFO:teuthology.orchestra.run.smithi002.stderr:Error EINVAL: invalid command 2022-03-08T18:38:11.468 INFO:teuthology.orchestra.run.smithi002.stderr:command terminated with exit code 22 2022-03-08T18:38:11.471 DEBUG:teuthology.orchestra.run:got remote process result: 22 2022-03-08T18:38:11.537 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:11.538823 I | op-mon: mon c is not yet running 2022-03-08T18:38:11.538 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:11.538856 I | op-mon: mons running: [a b] 2022-03-08T18:38:16.564 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:16.563452 I | op-mon: mon c is not yet running 2022-03-08T18:38:16.565 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:16.563494 I | op-mon: mons running: [a b] 2022-03-08T18:38:21.473 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch status -f json 2022-03-08T18:38:21.580 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:21.581813 I | op-mon: mon c is not yet running 2022-03-08T18:38:21.580 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:21.581846 I | op-mon: mons running: [a b] 2022-03-08T18:38:21.950 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:38:21.951 INFO:teuthology.orchestra.run.smithi002.stderr:no valid command found; 10 closest matches: 2022-03-08T18:38:21.951 INFO:teuthology.orchestra.run.smithi002.stderr:pg stat 2022-03-08T18:38:21.951 INFO:teuthology.orchestra.run.smithi002.stderr:pg getmap 2022-03-08T18:38:21.952 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump [all|summary|sum|delta|pools|osds|pgs|pgs_brief...] 2022-03-08T18:38:21.952 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_json [all|summary|sum|pools|osds|pgs...] 2022-03-08T18:38:21.952 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_pools_json 2022-03-08T18:38:21.952 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-pool [...] 2022-03-08T18:38:21.952 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-primary [] [...] 2022-03-08T18:38:21.953 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-osd [] [...] 2022-03-08T18:38:21.953 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls [] [...] 2022-03-08T18:38:21.953 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_stuck [inactive|unclean|stale|undersized|degraded...] [] 2022-03-08T18:38:21.953 INFO:teuthology.orchestra.run.smithi002.stderr:Error EINVAL: invalid command 2022-03-08T18:38:21.955 INFO:teuthology.orchestra.run.smithi002.stderr:command terminated with exit code 22 2022-03-08T18:38:21.957 DEBUG:teuthology.orchestra.run:got remote process result: 22 2022-03-08T18:38:27.154 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:27.155391 I | op-mon: mon c is not yet running 2022-03-08T18:38:27.154 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:27.155427 I | op-mon: mons running: [a b] 2022-03-08T18:38:31.959 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch status -f json 2022-03-08T18:38:32.462 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:32.464354 I | op-mon: mon c is not yet running 2022-03-08T18:38:32.463 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:32.464394 I | op-mon: mons running: [a b] 2022-03-08T18:38:32.872 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:38:32.872 INFO:teuthology.orchestra.run.smithi002.stderr:no valid command found; 10 closest matches: 2022-03-08T18:38:32.872 INFO:teuthology.orchestra.run.smithi002.stderr:pg stat 2022-03-08T18:38:32.873 INFO:teuthology.orchestra.run.smithi002.stderr:pg getmap 2022-03-08T18:38:32.873 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump [all|summary|sum|delta|pools|osds|pgs|pgs_brief...] 2022-03-08T18:38:32.873 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_json [all|summary|sum|pools|osds|pgs...] 2022-03-08T18:38:32.873 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_pools_json 2022-03-08T18:38:32.874 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-pool [...] 2022-03-08T18:38:32.874 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-primary [] [...] 2022-03-08T18:38:32.874 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-osd [] [...] 2022-03-08T18:38:32.875 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls [] [...] 2022-03-08T18:38:32.875 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_stuck [inactive|unclean|stale|undersized|degraded...] [] 2022-03-08T18:38:32.875 INFO:teuthology.orchestra.run.smithi002.stderr:Error EINVAL: invalid command 2022-03-08T18:38:32.876 DEBUG:teuthology.orchestra.run:got remote process result: 22 2022-03-08T18:38:32.877 INFO:teuthology.orchestra.run.smithi002.stderr:command terminated with exit code 22 2022-03-08T18:38:38.406 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:38.407795 I | op-mon: mon c is not yet running 2022-03-08T18:38:38.406 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:38.407825 I | op-mon: mons running: [a b] 2022-03-08T18:38:42.878 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch status -f json 2022-03-08T18:38:43.375 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:38:43.376 INFO:teuthology.orchestra.run.smithi002.stderr:no valid command found; 10 closest matches: 2022-03-08T18:38:43.376 INFO:teuthology.orchestra.run.smithi002.stderr:pg stat 2022-03-08T18:38:43.377 INFO:teuthology.orchestra.run.smithi002.stderr:pg getmap 2022-03-08T18:38:43.377 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump [all|summary|sum|delta|pools|osds|pgs|pgs_brief...] 2022-03-08T18:38:43.377 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_json [all|summary|sum|pools|osds|pgs...] 2022-03-08T18:38:43.377 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_pools_json 2022-03-08T18:38:43.377 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-pool [...] 2022-03-08T18:38:43.378 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-primary [] [...] 2022-03-08T18:38:43.378 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-osd [] [...] 2022-03-08T18:38:43.378 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls [] [...] 2022-03-08T18:38:43.379 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_stuck [inactive|unclean|stale|undersized|degraded...] [] 2022-03-08T18:38:43.379 INFO:teuthology.orchestra.run.smithi002.stderr:Error EINVAL: invalid command 2022-03-08T18:38:43.380 INFO:teuthology.orchestra.run.smithi002.stderr:command terminated with exit code 22 2022-03-08T18:38:43.383 DEBUG:teuthology.orchestra.run:got remote process result: 22 2022-03-08T18:38:43.423 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:43.425355 I | op-mon: mon c is not yet running 2022-03-08T18:38:43.423 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:43.425381 I | op-mon: mons running: [a b] 2022-03-08T18:38:48.462 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:48.464400 I | op-mon: mons running: [a b c] 2022-03-08T18:38:51.706 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:51.708221 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:38:51.707 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:51.708247 I | op-config: setting "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:38:51.957 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:51.958806 I | op-config: successfully set "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:38:51.957 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:51.958843 I | op-config: setting "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:38:52.170 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:52.172500 I | op-config: successfully set "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:38:52.171 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:52.172528 I | op-config: setting "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:38:52.391 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:52.392779 I | op-config: successfully set "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:38:52.392 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:52.392812 I | op-config: setting "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:38:52.618 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:52.619864 I | op-config: successfully set "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:38:52.618 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:52.619894 I | op-config: setting "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:38:52.837 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:52.838634 I | op-config: successfully set "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:38:52.837 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:52.838669 I | op-config: deleting "log file" option from the mon configuration database 2022-03-08T18:38:53.053 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:53.054760 I | op-config: successfully deleted "log file" option from the mon configuration database 2022-03-08T18:38:53.053 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:53.054811 I | cephclient: getting or creating ceph auth key "client.csi-rbd-provisioner" 2022-03-08T18:38:53.385 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch status -f json 2022-03-08T18:38:53.397 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:53.399539 I | cephclient: getting or creating ceph auth key "client.csi-rbd-node" 2022-03-08T18:38:53.810 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:38:53.810 INFO:teuthology.orchestra.run.smithi002.stderr:no valid command found; 10 closest matches: 2022-03-08T18:38:53.811 INFO:teuthology.orchestra.run.smithi002.stderr:pg stat 2022-03-08T18:38:53.811 INFO:teuthology.orchestra.run.smithi002.stderr:pg getmap 2022-03-08T18:38:53.811 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump [all|summary|sum|delta|pools|osds|pgs|pgs_brief...] 2022-03-08T18:38:53.811 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_json [all|summary|sum|pools|osds|pgs...] 2022-03-08T18:38:53.811 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_pools_json 2022-03-08T18:38:53.812 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-pool [...] 2022-03-08T18:38:53.812 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-primary [] [...] 2022-03-08T18:38:53.812 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-osd [] [...] 2022-03-08T18:38:53.812 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls [] [...] 2022-03-08T18:38:53.813 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_stuck [inactive|unclean|stale|undersized|degraded...] [] 2022-03-08T18:38:53.813 INFO:teuthology.orchestra.run.smithi002.stderr:Error EINVAL: invalid command 2022-03-08T18:38:53.813 INFO:teuthology.orchestra.run.smithi002.stderr:command terminated with exit code 22 2022-03-08T18:38:53.816 DEBUG:teuthology.orchestra.run:got remote process result: 22 2022-03-08T18:38:53.823 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:53.825691 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-provisioner" 2022-03-08T18:38:54.268 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:54.270149 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-node" 2022-03-08T18:38:54.664 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:54.666471 I | ceph-csi: created kubernetes csi secrets for cluster "rook-ceph" 2022-03-08T18:38:54.665 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:54.666511 I | cephclient: getting or creating ceph auth key "client.crash" 2022-03-08T18:38:55.053 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:55.055542 I | ceph-crashcollector-controller: created kubernetes crash collector secret for cluster "rook-ceph" 2022-03-08T18:38:55.294 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:55.295664 I | cephclient: successfully enabled msgr2 protocol 2022-03-08T18:38:55.294 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:55.295704 I | cephclient: create rbd-mirror bootstrap peer token "client.rbd-mirror-peer" 2022-03-08T18:38:55.294 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:55.295714 I | cephclient: getting or creating ceph auth key "client.rbd-mirror-peer" 2022-03-08T18:38:55.609 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:55.610877 I | cephclient: successfully created rbd-mirror bootstrap peer token for cluster "rook-ceph" 2022-03-08T18:38:55.647 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:55.649252 I | op-mgr: start running mgr 2022-03-08T18:38:55.647 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:55.649299 I | cephclient: getting or creating ceph auth key "mgr.a" 2022-03-08T18:38:57.232 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:57.234484 E | ceph-crashcollector-controller: node reconcile failed on op "unchanged": Operation cannot be fulfilled on deployments.apps "rook-ceph-crashcollector-smithi002": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T18:38:58.740 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:38:58.741291 E | ceph-crashcollector-controller: node reconcile failed on op "unchanged": Operation cannot be fulfilled on deployments.apps "rook-ceph-crashcollector-smithi137": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T18:39:03.817 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch status -f json 2022-03-08T18:39:04.484 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:39:04.485 INFO:teuthology.orchestra.run.smithi002.stderr:no valid command found; 10 closest matches: 2022-03-08T18:39:04.485 INFO:teuthology.orchestra.run.smithi002.stderr:pg stat 2022-03-08T18:39:04.485 INFO:teuthology.orchestra.run.smithi002.stderr:pg getmap 2022-03-08T18:39:04.486 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump [all|summary|sum|delta|pools|osds|pgs|pgs_brief...] 2022-03-08T18:39:04.486 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_json [all|summary|sum|pools|osds|pgs...] 2022-03-08T18:39:04.486 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_pools_json 2022-03-08T18:39:04.486 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-pool [...] 2022-03-08T18:39:04.487 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-primary [] [...] 2022-03-08T18:39:04.487 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-osd [] [...] 2022-03-08T18:39:04.487 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls [] [...] 2022-03-08T18:39:04.487 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_stuck [inactive|unclean|stale|undersized|degraded...] [] 2022-03-08T18:39:04.487 INFO:teuthology.orchestra.run.smithi002.stderr:Error EINVAL: invalid command 2022-03-08T18:39:04.488 INFO:teuthology.orchestra.run.smithi002.stderr:command terminated with exit code 22 2022-03-08T18:39:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 22 2022-03-08T18:39:08.267 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.269947 I | op-k8sutil: finished waiting for updated deployment "rook-ceph-mgr-a" 2022-03-08T18:39:08.272 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.272964 I | op-mgr: setting services to point to mgr "a" 2022-03-08T18:39:08.390 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.392277 I | op-osd: start running osds in namespace "rook-ceph" 2022-03-08T18:39:08.391 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.392325 W | op-osd: useAllNodes is set to false and no nodes, storageClassDevicesets or volumeSources are specified, no OSD pods are going to be created 2022-03-08T18:39:08.391 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.392345 I | op-osd: wait timeout for healthy OSDs during upgrade or restart is "10m0s" 2022-03-08T18:39:08.415 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.418313 I | op-osd: start provisioning the OSDs on PVCs, if needed 2022-03-08T18:39:08.422 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.425047 I | op-osd: no storageClassDeviceSets defined to configure OSDs on PVCs 2022-03-08T18:39:08.423 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.425080 I | op-osd: start provisioning the OSDs on nodes, if needed 2022-03-08T18:39:08.423 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.425088 I | op-osd: no nodes are defined for configuring OSDs on raw devices 2022-03-08T18:39:08.711 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.713176 I | op-osd: finished running OSDs in namespace "rook-ceph" 2022-03-08T18:39:08.711 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.713200 I | ceph-cluster-controller: done reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:39:08.721 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.724208 I | ceph-cluster-controller: enabling ceph mon monitoring goroutine for cluster "rook-ceph" 2022-03-08T18:39:08.722 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.724231 I | ceph-cluster-controller: enabling ceph osd monitoring goroutine for cluster "rook-ceph" 2022-03-08T18:39:08.722 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.724250 I | ceph-cluster-controller: enabling ceph status monitoring goroutine for cluster "rook-ceph" 2022-03-08T18:39:08.798 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.800772 I | op-k8sutil: ROOK_OBC_WATCH_OPERATOR_NAMESPACE="true" (configmap) 2022-03-08T18:39:08.798 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.800799 I | op-bucket-prov: ceph bucket provisioner launched watching for provisioner "rook-ceph.ceph.rook.io/bucket" 2022-03-08T18:39:08.802 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.802105 I | op-k8sutil: Reporting Event rook-ceph:rook-ceph Normal:ReconcileSucceeded:successfully configured CephCluster "rook-ceph/rook-ceph" 2022-03-08T18:39:08.802 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.802234 I | ceph-cluster-controller: reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:39:08.803 INFO:tasks.rook.operator.smithi002.stdout:I0308 18:39:08.802343 7 manager.go:118] objectbucket.io/provisioner-manager "msg"="starting provisioner" "name"="rook-ceph.ceph.rook.io/bucket" 2022-03-08T18:39:08.832 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.834487 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:39:08.843 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.845411 I | ceph-cluster-controller: detecting the ceph image version for image quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae... 2022-03-08T18:39:08.904 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:08.906968 I | op-mgr: successful modules: dashboard 2022-03-08T18:39:09.028 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:09.030420 W | cephclient: failed to enable mgr module "rook". trying again... 2022-03-08T18:39:09.085 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:09.088054 W | cephclient: failed to set mgr module mode "upmap". trying again... 2022-03-08T18:39:10.069 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:10.071898 I | op-mgr: successful modules: prometheus 2022-03-08T18:39:14.285 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:14.287675 W | cephclient: failed to enable mgr module "rook". trying again... 2022-03-08T18:39:14.380 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:14.382681 W | cephclient: failed to set mgr module mode "upmap". trying again... 2022-03-08T18:39:14.490 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch status -f json 2022-03-08T18:39:14.778 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:14.780287 I | ceph-cluster-controller: detected ceph image version: "17.0.0-10961 quincy" 2022-03-08T18:39:14.778 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:14.780311 I | ceph-cluster-controller: validating ceph version from provided image 2022-03-08T18:39:14.779 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:14.780321 W | ceph-cluster-controller: unsupported ceph version detected: "17.0.0-10961 quincy", pursuing 2022-03-08T18:39:14.807 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:14.809952 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:39:15.001 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:39:15.001 INFO:teuthology.orchestra.run.smithi002.stderr:no valid command found; 10 closest matches: 2022-03-08T18:39:15.001 INFO:teuthology.orchestra.run.smithi002.stderr:pg stat 2022-03-08T18:39:15.002 INFO:teuthology.orchestra.run.smithi002.stderr:pg getmap 2022-03-08T18:39:15.002 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump [all|summary|sum|delta|pools|osds|pgs|pgs_brief...] 2022-03-08T18:39:15.003 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_json [all|summary|sum|pools|osds|pgs...] 2022-03-08T18:39:15.003 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_pools_json 2022-03-08T18:39:15.003 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-pool [...] 2022-03-08T18:39:15.003 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-primary [] [...] 2022-03-08T18:39:15.003 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-osd [] [...] 2022-03-08T18:39:15.004 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls [] [...] 2022-03-08T18:39:15.004 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_stuck [inactive|unclean|stale|undersized|degraded...] [] 2022-03-08T18:39:15.004 INFO:teuthology.orchestra.run.smithi002.stderr:Error EINVAL: invalid command 2022-03-08T18:39:15.005 INFO:teuthology.orchestra.run.smithi002.stderr:command terminated with exit code 22 2022-03-08T18:39:15.007 DEBUG:teuthology.orchestra.run:got remote process result: 22 2022-03-08T18:39:15.207 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:15.209656 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:39:15.207 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:15.209964 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:39:15.458 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:15.460297 I | ceph-cluster-controller: cluster "rook-ceph": version "17.0.0-10961 quincy" detected for image "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae" 2022-03-08T18:39:15.573 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:15.575708 I | op-mon: start running mons 2022-03-08T18:39:15.616 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:15.617829 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:39:15.718 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:15.721126 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.109.10.123:6789","10.101.136.169:6789","10.110.64.66:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:39:15.763 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:15.765705 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:39:15.767 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:15.766007 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:39:16.704 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:16.707139 I | op-mon: targeting the mon count 3 2022-03-08T18:39:16.735 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:16.737582 I | op-config: setting "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:39:16.976 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:16.979320 I | op-config: successfully set "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:39:16.977 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:16.979350 I | op-config: setting "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:39:17.185 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:17.187269 I | op-config: successfully set "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:39:17.185 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:17.187296 I | op-config: setting "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:39:17.254 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:17.256009 E | ceph-crashcollector-controller: node reconcile failed on op "unchanged": Operation cannot be fulfilled on deployments.apps "rook-ceph-crashcollector-smithi066": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T18:39:17.266 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:17.268865 E | ceph-crashcollector-controller: node reconcile failed on op "unchanged": Operation cannot be fulfilled on deployments.apps "rook-ceph-crashcollector-smithi066": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T18:39:17.349 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:17.351551 E | ceph-crashcollector-controller: node reconcile failed on op "unchanged": Operation cannot be fulfilled on deployments.apps "rook-ceph-crashcollector-smithi066": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T18:39:17.413 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:17.414983 I | op-config: successfully set "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:39:17.413 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:17.415014 I | op-config: setting "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:39:17.645 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:17.646719 I | op-config: successfully set "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:39:17.646 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:17.646763 I | op-config: setting "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:39:17.878 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:17.880161 I | op-config: successfully set "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:39:17.878 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:17.880197 I | op-config: deleting "log file" option from the mon configuration database 2022-03-08T18:39:18.091 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:18.093742 I | op-config: successfully deleted "log file" option from the mon configuration database 2022-03-08T18:39:18.092 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:18.093777 I | op-mon: checking for basic quorum with existing mons 2022-03-08T18:39:18.201 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:18.203599 I | op-mon: mon "a" endpoint is [v2:10.101.136.169:3300,v1:10.101.136.169:6789] 2022-03-08T18:39:18.308 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:18.311101 I | op-mon: mon "b" endpoint is [v2:10.110.64.66:3300,v1:10.110.64.66:6789] 2022-03-08T18:39:19.027 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:19.029448 I | op-mon: mon "c" endpoint is [v2:10.109.10.123:3300,v1:10.109.10.123:6789] 2022-03-08T18:39:19.626 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:19.628916 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:39:19.678 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:19.680802 W | cephclient: failed to set mgr module mode "upmap". trying again... 2022-03-08T18:39:20.225 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:20.227779 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:39:20.229 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:20.228082 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:39:20.443 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:20.445523 I | op-mgr: successful modules: mgr module(s) from the spec 2022-03-08T18:39:20.680 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:20.682961 I | op-mon: deployment for mon rook-ceph-mon-a already exists. updating if needed 2022-03-08T18:39:20.691 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:20.692686 I | op-k8sutil: deployment "rook-ceph-mon-a" did not change, nothing to update 2022-03-08T18:39:20.691 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:20.692706 I | op-mon: waiting for mon quorum with [a b c] 2022-03-08T18:39:21.230 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:21.231604 I | op-mon: mons running: [a b c] 2022-03-08T18:39:21.463 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:21.466079 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:39:21.470 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:21.473020 I | op-mon: deployment for mon rook-ceph-mon-b already exists. updating if needed 2022-03-08T18:39:21.502 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:21.504811 I | op-k8sutil: deployment "rook-ceph-mon-b" did not change, nothing to update 2022-03-08T18:39:21.502 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:21.504843 I | op-mon: waiting for mon quorum with [a b c] 2022-03-08T18:39:21.830 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:21.832918 I | op-mon: mons running: [a b c] 2022-03-08T18:39:21.926 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:21.928700 I | cephclient: command failed for set rook backend. trying again... 2022-03-08T18:39:22.041 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:22.044410 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:39:22.080 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:22.083141 I | op-mon: deployment for mon rook-ceph-mon-c already exists. updating if needed 2022-03-08T18:39:22.094 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:22.095834 I | op-k8sutil: deployment "rook-ceph-mon-c" did not change, nothing to update 2022-03-08T18:39:22.094 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:22.095860 I | op-mon: waiting for mon quorum with [a b c] 2022-03-08T18:39:22.429 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:22.431838 I | op-mon: mons running: [a b c] 2022-03-08T18:39:22.666 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:22.666611 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:39:22.667 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:22.666639 I | op-mon: mons created: 3 2022-03-08T18:39:22.903 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:22.905379 I | op-mon: waiting for mon quorum with [a b c] 2022-03-08T18:39:23.030 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:23.032381 I | op-mon: mons running: [a b c] 2022-03-08T18:39:23.249 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:23.251937 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:39:23.250 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:23.251981 I | cephclient: getting or creating ceph auth key "client.csi-rbd-provisioner" 2022-03-08T18:39:23.495 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:23.497166 I | cephclient: getting or creating ceph auth key "client.csi-rbd-node" 2022-03-08T18:39:23.745 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:23.747442 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-provisioner" 2022-03-08T18:39:23.978 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:23.981385 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-node" 2022-03-08T18:39:24.658 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:24.660973 I | ceph-csi: created kubernetes csi secrets for cluster "rook-ceph" 2022-03-08T18:39:24.658 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:24.661005 I | cephclient: getting or creating ceph auth key "client.crash" 2022-03-08T18:39:24.970 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:24.969538 W | cephclient: failed to set mgr module mode "upmap". trying again... 2022-03-08T18:39:25.008 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch status -f json 2022-03-08T18:39:25.037 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:25.039657 I | ceph-crashcollector-controller: created kubernetes crash collector secret for cluster "rook-ceph" 2022-03-08T18:39:25.327 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:25.329967 I | cephclient: successfully enabled msgr2 protocol 2022-03-08T18:39:25.328 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:25.329999 I | cephclient: create rbd-mirror bootstrap peer token "client.rbd-mirror-peer" 2022-03-08T18:39:25.328 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:25.330005 I | cephclient: getting or creating ceph auth key "client.rbd-mirror-peer" 2022-03-08T18:39:25.487 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:39:25.488 INFO:teuthology.orchestra.run.smithi002.stderr:no valid command found; 10 closest matches: 2022-03-08T18:39:25.488 INFO:teuthology.orchestra.run.smithi002.stderr:pg stat 2022-03-08T18:39:25.488 INFO:teuthology.orchestra.run.smithi002.stderr:pg getmap 2022-03-08T18:39:25.488 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump [all|summary|sum|delta|pools|osds|pgs|pgs_brief...] 2022-03-08T18:39:25.488 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_json [all|summary|sum|pools|osds|pgs...] 2022-03-08T18:39:25.489 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_pools_json 2022-03-08T18:39:25.489 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-pool [...] 2022-03-08T18:39:25.489 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-primary [] [...] 2022-03-08T18:39:25.490 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls-by-osd [] [...] 2022-03-08T18:39:25.490 INFO:teuthology.orchestra.run.smithi002.stderr:pg ls [] [...] 2022-03-08T18:39:25.490 INFO:teuthology.orchestra.run.smithi002.stderr:pg dump_stuck [inactive|unclean|stale|undersized|degraded...] [] 2022-03-08T18:39:25.490 INFO:teuthology.orchestra.run.smithi002.stderr:Error EINVAL: invalid command 2022-03-08T18:39:25.491 INFO:teuthology.orchestra.run.smithi002.stderr:command terminated with exit code 22 2022-03-08T18:39:25.493 DEBUG:teuthology.orchestra.run:got remote process result: 22 2022-03-08T18:39:25.554 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:25.557271 I | cephclient: successfully created rbd-mirror bootstrap peer token for cluster "rook-ceph" 2022-03-08T18:39:25.601 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:25.604239 I | op-mgr: start running mgr 2022-03-08T18:39:25.602 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:25.604286 I | cephclient: getting or creating ceph auth key "mgr.a" 2022-03-08T18:39:26.056 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:26.058679 I | op-mgr: deployment for mgr rook-ceph-mgr-a already exists. updating if needed 2022-03-08T18:39:26.065 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:26.068249 I | op-k8sutil: deployment "rook-ceph-mgr-a" did not change, nothing to update 2022-03-08T18:39:26.068 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:26.070775 I | op-mgr: setting services to point to mgr "a" 2022-03-08T18:39:26.835 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:26.837394 I | op-osd: start running osds in namespace "rook-ceph" 2022-03-08T18:39:26.835 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:26.837429 W | op-osd: useAllNodes is set to false and no nodes, storageClassDevicesets or volumeSources are specified, no OSD pods are going to be created 2022-03-08T18:39:26.835 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:26.837443 I | op-osd: wait timeout for healthy OSDs during upgrade or restart is "10m0s" 2022-03-08T18:39:26.859 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:26.862347 I | op-osd: start provisioning the OSDs on PVCs, if needed 2022-03-08T18:39:27.024 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:27.026229 I | op-osd: no storageClassDeviceSets defined to configure OSDs on PVCs 2022-03-08T18:39:27.024 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:27.026266 I | op-osd: start provisioning the OSDs on nodes, if needed 2022-03-08T18:39:27.025 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:27.026275 I | op-osd: no nodes are defined for configuring OSDs on raw devices 2022-03-08T18:39:27.278 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:27.281128 I | cephclient: command failed for set rook backend. trying again... 2022-03-08T18:39:27.499 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:27.501952 I | op-mgr: successful modules: mgr module(s) from the spec 2022-03-08T18:39:27.502 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:27.503521 I | op-mgr: successful modules: prometheus 2022-03-08T18:39:27.502 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:27.503898 I | op-mgr: successful modules: dashboard 2022-03-08T18:39:27.689 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:27.691481 I | op-osd: finished running OSDs in namespace "rook-ceph" 2022-03-08T18:39:27.689 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:27.691512 I | ceph-cluster-controller: done reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:39:27.777 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:27.780269 W | cephclient: failed to set mgr module mode "upmap". trying again... 2022-03-08T18:39:28.913 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:28.915426 I | cephclient: command failed for set rook backend. trying again... 2022-03-08T18:39:30.441 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:30.443464 I | op-mgr: successful modules: balancer 2022-03-08T18:39:32.743 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:32.745831 I | cephclient: action set rook backend succeeded on attempt 2 2022-03-08T18:39:32.743 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:32.745869 I | op-mgr: successful modules: orchestrator modules 2022-03-08T18:39:33.118 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:33.120733 I | op-mgr: successful modules: balancer 2022-03-08T18:39:34.288 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:34.290569 I | cephclient: action set rook backend succeeded on attempt 1 2022-03-08T18:39:34.288 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:34.290596 I | op-mgr: successful modules: orchestrator modules 2022-03-08T18:39:35.495 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch status -f json 2022-03-08T18:39:36.126 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:39:36.127 INFO:teuthology.orchestra.run.smithi002.stdout:{"available": true, "backend": "rook"} 2022-03-08T18:39:36.151 INFO:tasks.rook: mgr/rook orchestrator is active 2022-03-08T18:39:36.152 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph config set mgr mgr/rook/storage_class scratch 2022-03-08T18:39:36.777 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch apply osd --all-available-devices 2022-03-08T18:39:37.695 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:37.697683 I | ceph-cluster-controller: CR has changed for "rook-ceph". diff=  v1.ClusterSpec{ 2022-03-08T18:39:37.696 INFO:tasks.rook.operator.smithi002.stdout:   CephVersion: {Image: "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", AllowUnsupported: true}, 2022-03-08T18:39:37.696 INFO:tasks.rook.operator.smithi002.stdout:   Storage: v1.StorageScopeSpec{ 2022-03-08T18:39:37.696 INFO:tasks.rook.operator.smithi002.stdout:   ... // 3 identical fields 2022-03-08T18:39:37.697 INFO:tasks.rook.operator.smithi002.stdout:   Config: nil, 2022-03-08T18:39:37.697 INFO:tasks.rook.operator.smithi002.stdout:   Selection: {}, 2022-03-08T18:39:37.697 INFO:tasks.rook.operator.smithi002.stdout:-  StorageClassDeviceSets: nil, 2022-03-08T18:39:37.697 INFO:teuthology.orchestra.run.smithi002.stdout:['Success'] 2022-03-08T18:39:37.704 INFO:tasks.rook.operator.smithi002.stdout:+  StorageClassDeviceSets: []v1.StorageClassDeviceSet{ 2022-03-08T18:39:37.704 INFO:tasks.rook.operator.smithi002.stdout:+  { 2022-03-08T18:39:37.704 INFO:tasks.rook.operator.smithi002.stdout:+  Name: "smithi002-lv-1", 2022-03-08T18:39:37.705 INFO:tasks.rook.operator.smithi002.stdout:+  Count: 1, 2022-03-08T18:39:37.705 INFO:tasks.rook.operator.smithi002.stdout:+  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{...}}, 2022-03-08T18:39:37.705 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:39:37.705 INFO:tasks.rook.operator.smithi002.stdout:+  { 2022-03-08T18:39:37.706 INFO:tasks.rook.operator.smithi002.stdout:+  Name: "smithi002-lv-2", 2022-03-08T18:39:37.706 INFO:tasks.rook.operator.smithi002.stdout:+  Count: 1, 2022-03-08T18:39:37.707 INFO:tasks.rook.operator.smithi002.stdout:+  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{...}}, 2022-03-08T18:39:37.707 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:39:37.707 INFO:tasks.rook.operator.smithi002.stdout:+  { 2022-03-08T18:39:37.707 INFO:tasks.rook.operator.smithi002.stdout:+  Name: "smithi002-lv-3", 2022-03-08T18:39:37.708 INFO:tasks.rook.operator.smithi002.stdout:+  Count: 1, 2022-03-08T18:39:37.708 INFO:tasks.rook.operator.smithi002.stdout:+  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{...}}, 2022-03-08T18:39:37.708 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:39:37.708 INFO:tasks.rook.operator.smithi002.stdout:+  { 2022-03-08T18:39:37.709 INFO:tasks.rook.operator.smithi002.stdout:+  Name: "smithi002-lv-4", 2022-03-08T18:39:37.709 INFO:tasks.rook.operator.smithi002.stdout:+  Count: 1, 2022-03-08T18:39:37.710 INFO:tasks.rook.operator.smithi002.stdout:+  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{...}}, 2022-03-08T18:39:37.710 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:39:37.710 INFO:tasks.rook.operator.smithi002.stdout:+  { 2022-03-08T18:39:37.710 INFO:tasks.rook.operator.smithi002.stdout:+  Name: "smithi066-lv-1", 2022-03-08T18:39:37.711 INFO:tasks.rook.operator.smithi002.stdout:+  Count: 1, 2022-03-08T18:39:37.711 INFO:tasks.rook.operator.smithi002.stdout:+  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{...}}, 2022-03-08T18:39:37.711 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:39:37.711 INFO:tasks.rook.operator.smithi002.stdout:+  { 2022-03-08T18:39:37.711 INFO:tasks.rook.operator.smithi002.stdout:+  Name: "smithi066-lv-2", 2022-03-08T18:39:37.712 INFO:tasks.rook.operator.smithi002.stdout:+  Count: 1, 2022-03-08T18:39:37.712 INFO:tasks.rook.operator.smithi002.stdout:+  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{...}}, 2022-03-08T18:39:37.712 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:39:37.712 INFO:tasks.rook.operator.smithi002.stdout:+  { 2022-03-08T18:39:37.713 INFO:tasks.rook.operator.smithi002.stdout:+  Name: "smithi066-lv-3", 2022-03-08T18:39:37.713 INFO:tasks.rook.operator.smithi002.stdout:+  Count: 1, 2022-03-08T18:39:37.713 INFO:tasks.rook.operator.smithi002.stdout:+  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{...}}, 2022-03-08T18:39:37.713 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:39:37.713 INFO:tasks.rook.operator.smithi002.stdout:+  { 2022-03-08T18:39:37.714 INFO:tasks.rook.operator.smithi002.stdout:+  Name: "smithi066-lv-4", 2022-03-08T18:39:37.714 INFO:tasks.rook.operator.smithi002.stdout:+  Count: 1, 2022-03-08T18:39:37.714 INFO:tasks.rook.operator.smithi002.stdout:+  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{...}}, 2022-03-08T18:39:37.714 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:39:37.715 INFO:tasks.rook.operator.smithi002.stdout:+  { 2022-03-08T18:39:37.715 INFO:tasks.rook.operator.smithi002.stdout:+  Name: "smithi137-lv-1", 2022-03-08T18:39:37.715 INFO:tasks.rook.operator.smithi002.stdout:+  Count: 1, 2022-03-08T18:39:37.715 INFO:tasks.rook.operator.smithi002.stdout:+  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{...}}, 2022-03-08T18:39:37.715 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:39:37.716 INFO:tasks.rook.operator.smithi002.stdout:+  { 2022-03-08T18:39:37.716 INFO:tasks.rook.operator.smithi002.stdout:+  Name: "smithi137-lv-2", 2022-03-08T18:39:37.716 INFO:tasks.rook.operator.smithi002.stdout:+  Count: 1, 2022-03-08T18:39:37.716 INFO:tasks.rook.operator.smithi002.stdout:+  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{...}}, 2022-03-08T18:39:37.716 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:39:37.717 INFO:tasks.rook.operator.smithi002.stdout:+  { 2022-03-08T18:39:37.717 INFO:tasks.rook.operator.smithi002.stdout:+  Name: "smithi137-lv-3", 2022-03-08T18:39:37.717 INFO:tasks.rook.operator.smithi002.stdout:+  Count: 1, 2022-03-08T18:39:37.717 INFO:tasks.rook.operator.smithi002.stdout:+  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{...}}, 2022-03-08T18:39:37.718 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:39:37.718 INFO:tasks.rook.operator.smithi002.stdout:+  { 2022-03-08T18:39:37.718 INFO:tasks.rook.operator.smithi002.stdout:+  Name: "smithi137-lv-4", 2022-03-08T18:39:37.718 INFO:tasks.rook.operator.smithi002.stdout:+  Count: 1, 2022-03-08T18:39:37.718 INFO:tasks.rook.operator.smithi002.stdout:+  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{...}}, 2022-03-08T18:39:37.719 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:39:37.719 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:39:37.719 INFO:tasks.rook.operator.smithi002.stdout:   }, 2022-03-08T18:39:37.719 INFO:tasks.rook.operator.smithi002.stdout:   Annotations: nil, 2022-03-08T18:39:37.720 INFO:tasks.rook.operator.smithi002.stdout:   Labels: nil, 2022-03-08T18:39:37.721 INFO:tasks.rook.operator.smithi002.stdout:   ... // 20 identical fields 2022-03-08T18:39:37.721 INFO:tasks.rook.operator.smithi002.stdout:  } 2022-03-08T18:39:37.721 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:37.697818 I | ceph-cluster-controller: reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:39:37.722 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:37.718742 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:39:37.724 INFO:tasks.rook:Waiting for 12 OSDs 2022-03-08T18:39:37.725 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:39:37.733 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:37.736064 I | ceph-cluster-controller: detecting the ceph image version for image quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae... 2022-03-08T18:39:38.243 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:39:38.270 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":4,"num_osds":0,"num_up_osds":0,"osd_up_since":0,"num_in_osds":0,"osd_in_since":0,"num_remapped_pgs":0} 2022-03-08T18:39:38.271 INFO:tasks.rook: have 0/12 OSDs 2022-03-08T18:39:43.713 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:43.715996 I | ceph-cluster-controller: detected ceph image version: "17.0.0-10961 quincy" 2022-03-08T18:39:43.713 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:43.716020 I | ceph-cluster-controller: validating ceph version from provided image 2022-03-08T18:39:43.714 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:43.716033 W | ceph-cluster-controller: unsupported ceph version detected: "17.0.0-10961 quincy", pursuing 2022-03-08T18:39:43.732 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:43.735834 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:39:44.089 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:44.091681 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:39:44.093 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:44.092024 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:39:44.486 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:44.489602 I | ceph-cluster-controller: cluster "rook-ceph": version "17.0.0-10961 quincy" detected for image "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae" 2022-03-08T18:39:44.619 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:44.622446 I | op-mon: start running mons 2022-03-08T18:39:44.693 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:44.696205 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:39:44.754 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:44.756947 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:39:44.783 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:44.785629 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:39:44.786 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:44.785943 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:39:45.634 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:45.637038 I | op-mon: targeting the mon count 3 2022-03-08T18:39:45.670 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:45.673219 I | ceph-cluster-controller: Cluster update requested. CANCELLING CURRENT ORCHESTRATION 2022-03-08T18:39:46.670 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:46.673505 I | ceph-cluster-controller: reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:39:46.678 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:46.681277 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:39:46.700 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:46.703132 I | ceph-cluster-controller: detecting the ceph image version for image quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae... 2022-03-08T18:39:46.723 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:46.726677 I | op-k8sutil: Removing previous job rook-ceph-detect-version to start a new one 2022-03-08T18:39:46.730 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:46.733182 I | op-k8sutil: batch job rook-ceph-detect-version still exists 2022-03-08T18:39:48.272 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:39:48.734 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:48.737481 I | op-k8sutil: batch job rook-ceph-detect-version deleted 2022-03-08T18:39:48.864 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:39:48.887 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":4,"num_osds":0,"num_up_osds":0,"osd_up_since":0,"num_in_osds":0,"osd_in_since":0,"num_remapped_pgs":0} 2022-03-08T18:39:48.888 INFO:tasks.rook: have 0/12 OSDs 2022-03-08T18:39:54.745 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:54.747867 I | ceph-cluster-controller: detected ceph image version: "17.0.0-10961 quincy" 2022-03-08T18:39:54.745 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:54.747897 I | ceph-cluster-controller: validating ceph version from provided image 2022-03-08T18:39:54.746 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:54.747910 W | ceph-cluster-controller: unsupported ceph version detected: "17.0.0-10961 quincy", pursuing 2022-03-08T18:39:55.035 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:55.038948 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:39:55.040 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:55.041285 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:39:55.040 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:55.041512 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:39:55.456 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:55.459369 I | ceph-cluster-controller: cluster "rook-ceph": version "17.0.0-10961 quincy" detected for image "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae" 2022-03-08T18:39:55.579 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:55.582608 I | op-mon: start running mons 2022-03-08T18:39:55.605 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:55.608205 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:39:55.647 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:55.650550 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:39:55.665 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:55.668356 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:39:55.668 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:55.668706 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:39:56.532 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:56.535581 I | op-mon: targeting the mon count 3 2022-03-08T18:39:56.537 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:56.540772 I | op-config: setting "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:39:56.908 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:56.911452 I | op-config: successfully set "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:39:56.909 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:56.911483 I | op-config: setting "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:39:57.254 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:57.257026 I | op-config: successfully set "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:39:57.254 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:57.257058 I | op-config: setting "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:39:57.591 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:57.594046 I | op-config: successfully set "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:39:57.591 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:57.594080 I | op-config: setting "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:39:57.940 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:57.942938 I | op-config: successfully set "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:39:57.940 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:57.942971 I | op-config: setting "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:39:58.274 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:58.277460 I | op-config: successfully set "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:39:58.275 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:58.277489 I | op-config: deleting "log file" option from the mon configuration database 2022-03-08T18:39:58.608 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:58.611178 I | op-config: successfully deleted "log file" option from the mon configuration database 2022-03-08T18:39:58.608 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:58.611319 I | op-mon: checking for basic quorum with existing mons 2022-03-08T18:39:58.756 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:58.758966 I | op-mon: mon "b" endpoint is [v2:10.110.64.66:3300,v1:10.110.64.66:6789] 2022-03-08T18:39:58.890 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:39:58.932 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:58.935443 I | op-mon: mon "c" endpoint is [v2:10.109.10.123:3300,v1:10.109.10.123:6789] 2022-03-08T18:39:59.097 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:59.100229 I | op-mon: mon "a" endpoint is [v2:10.101.136.169:3300,v1:10.101.136.169:6789] 2022-03-08T18:39:59.429 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:39:59.448 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":4,"num_osds":0,"num_up_osds":0,"osd_up_since":0,"num_in_osds":0,"osd_in_since":0,"num_remapped_pgs":0} 2022-03-08T18:39:59.448 INFO:tasks.rook: have 0/12 OSDs 2022-03-08T18:39:59.615 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:39:59.618042 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:40:00.213 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:00.216689 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:40:00.214 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:00.217003 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:40:00.654 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:00.657873 I | op-mon: deployment for mon rook-ceph-mon-b already exists. updating if needed 2022-03-08T18:40:00.664 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:00.666985 I | op-k8sutil: deployment "rook-ceph-mon-b" did not change, nothing to update 2022-03-08T18:40:00.664 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:00.667013 I | op-mon: waiting for mon quorum with [b c a] 2022-03-08T18:40:01.231 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:01.234183 I | op-mon: mons running: [b c a] 2022-03-08T18:40:01.653 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:01.655082 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:40:01.657 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:01.661167 I | op-mon: deployment for mon rook-ceph-mon-c already exists. updating if needed 2022-03-08T18:40:01.668 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:01.670826 I | op-k8sutil: deployment "rook-ceph-mon-c" did not change, nothing to update 2022-03-08T18:40:01.668 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:01.670854 I | op-mon: waiting for mon quorum with [b c a] 2022-03-08T18:40:01.819 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:01.821882 I | op-mon: mons running: [b c a] 2022-03-08T18:40:02.243 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:02.246409 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:40:02.278 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:02.281950 I | op-mon: deployment for mon rook-ceph-mon-a already exists. updating if needed 2022-03-08T18:40:02.289 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:02.292235 I | op-k8sutil: deployment "rook-ceph-mon-a" did not change, nothing to update 2022-03-08T18:40:02.289 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:02.292277 I | op-mon: waiting for mon quorum with [b c a] 2022-03-08T18:40:02.417 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:02.420126 I | op-mon: mons running: [b c a] 2022-03-08T18:40:02.837 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:02.840958 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:40:02.838 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:02.840985 I | op-mon: mons created: 3 2022-03-08T18:40:03.236 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:03.238885 I | op-mon: waiting for mon quorum with [b c a] 2022-03-08T18:40:03.259 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:03.262275 I | op-mon: mons running: [b c a] 2022-03-08T18:40:03.676 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:03.678370 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:40:03.676 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:03.678427 I | cephclient: getting or creating ceph auth key "client.csi-rbd-provisioner" 2022-03-08T18:40:04.075 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:04.077684 I | cephclient: getting or creating ceph auth key "client.csi-rbd-node" 2022-03-08T18:40:04.463 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:04.465972 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-provisioner" 2022-03-08T18:40:04.872 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:04.874875 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-node" 2022-03-08T18:40:05.329 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:05.332637 I | ceph-csi: created kubernetes csi secrets for cluster "rook-ceph" 2022-03-08T18:40:05.330 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:05.332666 I | cephclient: getting or creating ceph auth key "client.crash" 2022-03-08T18:40:05.760 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:05.763498 I | ceph-crashcollector-controller: created kubernetes crash collector secret for cluster "rook-ceph" 2022-03-08T18:40:06.182 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:06.185011 I | cephclient: successfully enabled msgr2 protocol 2022-03-08T18:40:06.182 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:06.185051 I | cephclient: create rbd-mirror bootstrap peer token "client.rbd-mirror-peer" 2022-03-08T18:40:06.183 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:06.185059 I | cephclient: getting or creating ceph auth key "client.rbd-mirror-peer" 2022-03-08T18:40:06.594 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:06.597580 I | cephclient: successfully created rbd-mirror bootstrap peer token for cluster "rook-ceph" 2022-03-08T18:40:06.614 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:06.618171 I | ceph-spec: controller will reconcile resource "cluster-peer-token-rook-ceph" based on patch: [redacted patch details due to potentially sensitive content] 2022-03-08T18:40:06.629 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:06.630684 I | op-mgr: start running mgr 2022-03-08T18:40:06.629 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:06.630723 I | cephclient: getting or creating ceph auth key "mgr.a" 2022-03-08T18:40:07.142 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:07.145641 I | op-mgr: deployment for mgr rook-ceph-mgr-a already exists. updating if needed 2022-03-08T18:40:07.153 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:07.156788 I | op-k8sutil: deployment "rook-ceph-mgr-a" did not change, nothing to update 2022-03-08T18:40:07.156 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:07.158890 I | op-mgr: setting services to point to mgr "a" 2022-03-08T18:40:07.336 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:07.339327 I | op-osd: start running osds in namespace "rook-ceph" 2022-03-08T18:40:07.339 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:07.339367 I | op-osd: wait timeout for healthy OSDs during upgrade or restart is "10m0s" 2022-03-08T18:40:07.361 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:07.364331 I | op-osd: start provisioning the OSDs on PVCs, if needed 2022-03-08T18:40:07.364 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:07.366373 I | op-osd: creating 1 new PVCs for device set "smithi002-lv-1" 2022-03-08T18:40:07.509 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:07.512873 I | op-osd: successfully provisioned PVC "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:40:07.510 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:07.512902 I | op-osd: creating 1 new PVCs for device set "smithi002-lv-2" 2022-03-08T18:40:07.748 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:07.750753 I | op-osd: successfully provisioned PVC "smithi002-lv-2-data-09sjq5" 2022-03-08T18:40:07.749 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:07.750794 I | op-osd: creating 1 new PVCs for device set "smithi002-lv-3" 2022-03-08T18:40:07.909 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:07.913098 I | op-osd: successfully provisioned PVC "smithi002-lv-3-data-05dmhp" 2022-03-08T18:40:07.910 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:07.913151 I | op-osd: creating 1 new PVCs for device set "smithi002-lv-4" 2022-03-08T18:40:08.127 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.130792 I | op-osd: successfully provisioned PVC "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:40:08.128 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.130830 I | op-osd: creating 1 new PVCs for device set "smithi066-lv-1" 2022-03-08T18:40:08.336 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.340045 I | op-osd: successfully provisioned PVC "smithi066-lv-1-data-0drn25" 2022-03-08T18:40:08.337 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.340099 I | op-osd: creating 1 new PVCs for device set "smithi066-lv-2" 2022-03-08T18:40:08.507 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.510882 I | op-osd: successfully provisioned PVC "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:40:08.508 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.510918 I | op-osd: creating 1 new PVCs for device set "smithi066-lv-3" 2022-03-08T18:40:08.564 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.567465 I | op-mgr: successful modules: mgr module(s) from the spec 2022-03-08T18:40:08.569 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.569324 I | op-mgr: successful modules: prometheus 2022-03-08T18:40:08.622 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.625279 I | op-mgr: successful modules: dashboard 2022-03-08T18:40:08.707 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.710425 I | op-osd: successfully provisioned PVC "smithi066-lv-3-data-0xl876" 2022-03-08T18:40:08.707 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.710465 I | op-osd: creating 1 new PVCs for device set "smithi066-lv-4" 2022-03-08T18:40:08.907 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.910616 I | op-osd: successfully provisioned PVC "smithi066-lv-4-data-04lwmk" 2022-03-08T18:40:08.908 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.910650 I | op-osd: creating 1 new PVCs for device set "smithi137-lv-1" 2022-03-08T18:40:08.926 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:08.929812 I | op-mgr: successful modules: balancer 2022-03-08T18:40:09.107 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:09.109911 I | op-osd: successfully provisioned PVC "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:40:09.107 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:09.109949 I | op-osd: creating 1 new PVCs for device set "smithi137-lv-2" 2022-03-08T18:40:09.306 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:09.309761 I | op-osd: successfully provisioned PVC "smithi137-lv-2-data-0whzhj" 2022-03-08T18:40:09.307 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:09.309793 I | op-osd: creating 1 new PVCs for device set "smithi137-lv-3" 2022-03-08T18:40:09.451 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:40:09.511 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:09.511437 I | op-osd: successfully provisioned PVC "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:40:09.512 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:09.511473 I | op-osd: creating 1 new PVCs for device set "smithi137-lv-4" 2022-03-08T18:40:09.710 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:09.712820 I | op-osd: successfully provisioned PVC "smithi137-lv-4-data-0zchnd" 2022-03-08T18:40:09.729 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:09.732623 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:40:09.993 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:40:10.013 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":5,"num_osds":0,"num_up_osds":0,"osd_up_since":0,"num_in_osds":0,"osd_in_since":0,"num_remapped_pgs":0} 2022-03-08T18:40:10.014 INFO:tasks.rook: have 0/12 OSDs 2022-03-08T18:40:10.100 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:10.103965 I | op-mgr: successful modules: orchestrator modules 2022-03-08T18:40:10.210 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:10.212170 I | op-osd: started OSD provisioning job for PVC "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:40:10.211 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:10.212204 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-2-data-09sjq5" 2022-03-08T18:40:11.262 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:11.265662 I | op-osd: started OSD provisioning job for PVC "smithi002-lv-2-data-09sjq5" 2022-03-08T18:40:11.262 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:11.265699 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-3-data-05dmhp" 2022-03-08T18:40:12.225 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:12.228506 I | op-osd: started OSD provisioning job for PVC "smithi002-lv-3-data-05dmhp" 2022-03-08T18:40:12.225 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:12.228545 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:40:12.426 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:12.430107 I | op-osd: started OSD provisioning job for PVC "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:40:12.427 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:12.430134 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-1-data-0drn25" 2022-03-08T18:40:12.632 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:12.635993 I | op-osd: started OSD provisioning job for PVC "smithi066-lv-1-data-0drn25" 2022-03-08T18:40:12.633 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:12.636029 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:40:13.517 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:13.521041 I | op-osd: started OSD provisioning job for PVC "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:40:13.518 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:13.521084 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-3-data-0xl876" 2022-03-08T18:40:14.575 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:14.578012 I | op-osd: started OSD provisioning job for PVC "smithi066-lv-3-data-0xl876" 2022-03-08T18:40:14.575 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:14.578050 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-4-data-04lwmk" 2022-03-08T18:40:15.337 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:15.340813 I | op-osd: started OSD provisioning job for PVC "smithi066-lv-4-data-04lwmk" 2022-03-08T18:40:15.474 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:15.340850 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:40:15.888 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:15.891422 I | op-osd: started OSD provisioning job for PVC "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:40:15.888 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:15.891462 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-2-data-0whzhj" 2022-03-08T18:40:16.298 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:16.301649 I | op-osd: started OSD provisioning job for PVC "smithi137-lv-2-data-0whzhj" 2022-03-08T18:40:16.299 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:16.301683 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:40:16.511 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:16.514288 I | op-osd: started OSD provisioning job for PVC "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:40:16.511 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:16.514322 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-4-data-0zchnd" 2022-03-08T18:40:17.694 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.696669 I | op-osd: started OSD provisioning job for PVC "smithi137-lv-4-data-0zchnd" 2022-03-08T18:40:17.694 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.696708 I | op-osd: start provisioning the OSDs on nodes, if needed 2022-03-08T18:40:17.694 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.696715 I | op-osd: no nodes are defined for configuring OSDs on raw devices 2022-03-08T18:40:17.777 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.780027 I | op-osd: OSD orchestration status for PVC smithi002-lv-1-data-0cg8pd is "starting" 2022-03-08T18:40:17.778 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.780054 I | op-osd: OSD orchestration status for PVC smithi002-lv-2-data-09sjq5 is "starting" 2022-03-08T18:40:17.778 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.780064 I | op-osd: OSD orchestration status for PVC smithi002-lv-3-data-05dmhp is "starting" 2022-03-08T18:40:17.778 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.780071 I | op-osd: OSD orchestration status for PVC smithi002-lv-4-data-0pb5hz is "starting" 2022-03-08T18:40:17.779 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.780079 I | op-osd: OSD orchestration status for PVC smithi066-lv-1-data-0drn25 is "starting" 2022-03-08T18:40:17.779 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.780086 I | op-osd: OSD orchestration status for PVC smithi066-lv-2-data-0jsgjv is "starting" 2022-03-08T18:40:17.779 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.780096 I | op-osd: OSD orchestration status for PVC smithi066-lv-3-data-0xl876 is "starting" 2022-03-08T18:40:17.779 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.780116 I | op-osd: OSD orchestration status for PVC smithi066-lv-4-data-04lwmk is "starting" 2022-03-08T18:40:17.780 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.780131 I | op-osd: OSD orchestration status for PVC smithi137-lv-1-data-0cmhmp is "starting" 2022-03-08T18:40:17.780 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.780143 I | op-osd: OSD orchestration status for PVC smithi137-lv-2-data-0whzhj is "starting" 2022-03-08T18:40:17.780 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.780157 I | op-osd: OSD orchestration status for PVC smithi137-lv-3-data-0kg6xk is "starting" 2022-03-08T18:40:17.780 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:17.780169 I | op-osd: OSD orchestration status for PVC smithi137-lv-4-data-0zchnd is "starting" 2022-03-08T18:40:20.015 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:40:21.845 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:40:21.874 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":5,"num_osds":0,"num_up_osds":0,"osd_up_since":0,"num_in_osds":0,"osd_in_since":0,"num_remapped_pgs":0} 2022-03-08T18:40:21.875 INFO:tasks.rook: have 0/12 OSDs 2022-03-08T18:40:25.531 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:25.535360 I | op-osd: OSD orchestration status for node smithi066-lv-1-data-0drn25 is "orchestrating" 2022-03-08T18:40:26.650 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:26.652190 I | op-osd: OSD orchestration status for PVC smithi066-lv-1-data-0drn25 is "orchestrating" 2022-03-08T18:40:28.773 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:28.776959 I | op-osd: OSD orchestration status for node smithi002-lv-1-data-0cg8pd is "orchestrating" 2022-03-08T18:40:29.507 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:29.511352 I | op-osd: OSD orchestration status for PVC smithi002-lv-1-data-0cg8pd is "orchestrating" 2022-03-08T18:40:30.348 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:30.351789 I | op-osd: OSD orchestration status for node smithi137-lv-1-data-0cmhmp is "orchestrating" 2022-03-08T18:40:30.742 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:30.746239 I | op-osd: OSD orchestration status for PVC smithi137-lv-1-data-0cmhmp is "orchestrating" 2022-03-08T18:40:31.876 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:40:32.633 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:32.637617 I | op-osd: OSD orchestration status for node smithi066-lv-2-data-0jsgjv is "orchestrating" 2022-03-08T18:40:32.634 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:32.637698 I | op-osd: OSD orchestration status for node smithi066-lv-3-data-0xl876 is "orchestrating" 2022-03-08T18:40:33.102 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:40:33.119 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":6,"num_osds":1,"num_up_osds":0,"osd_up_since":0,"num_in_osds":1,"osd_in_since":1646764830,"num_remapped_pgs":0} 2022-03-08T18:40:33.120 INFO:tasks.rook: have 0/12 OSDs 2022-03-08T18:40:33.275 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:33.279888 I | op-osd: OSD orchestration status for PVC smithi066-lv-3-data-0xl876 is "orchestrating" 2022-03-08T18:40:33.290 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:33.294486 I | op-osd: OSD orchestration status for PVC smithi066-lv-2-data-0jsgjv is "orchestrating" 2022-03-08T18:40:34.051 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:34.054808 I | op-osd: OSD orchestration status for node smithi002-lv-2-data-09sjq5 is "orchestrating" 2022-03-08T18:40:34.051 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:34.054902 I | op-osd: OSD orchestration status for node smithi002-lv-4-data-0pb5hz is "orchestrating" 2022-03-08T18:40:35.065 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:35.069080 I | op-osd: OSD orchestration status for PVC smithi002-lv-4-data-0pb5hz is "orchestrating" 2022-03-08T18:40:35.065 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:35.069135 I | op-osd: OSD orchestration status for PVC smithi002-lv-2-data-09sjq5 is "orchestrating" 2022-03-08T18:40:35.484 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:35.488495 I | op-osd: OSD orchestration status for node smithi066-lv-4-data-04lwmk is "orchestrating" 2022-03-08T18:40:35.871 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:35.875542 I | op-osd: OSD orchestration status for PVC smithi066-lv-4-data-04lwmk is "orchestrating" 2022-03-08T18:40:36.039 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:36.042750 I | op-osd: OSD orchestration status for node smithi002-lv-3-data-05dmhp is "orchestrating" 2022-03-08T18:40:36.173 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:36.177037 I | op-osd: OSD orchestration status for node smithi137-lv-2-data-0whzhj is "orchestrating" 2022-03-08T18:40:36.454 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:36.458457 I | op-osd: OSD orchestration status for PVC smithi002-lv-3-data-05dmhp is "orchestrating" 2022-03-08T18:40:36.613 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:36.606325 I | op-osd: OSD orchestration status for PVC smithi002-lv-4-data-0pb5hz is "failed" 2022-03-08T18:40:36.614 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:36.606377 E | op-osd: failed to provision OSD(s) on PVC smithi002-lv-4-data-0pb5hz. &{OSDs:[] Status:failed PvcBackedOSD:true Message:failed to configure devices: failed to initialize devices on PVC: failed to run ceph-volume. Running command: /usr/bin/ceph-authtool --gen-print-key 2022-03-08T18:40:36.614 INFO:tasks.rook.operator.smithi002.stdout:Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 43fe8d67-6a36-4902-ad46-836d9bdfb8b1 2022-03-08T18:40:36.614 INFO:tasks.rook.operator.smithi002.stdout: stderr: unable to get monitor info from DNS SRV with service name: ceph-mon 2022-03-08T18:40:36.614 INFO:tasks.rook.operator.smithi002.stdout: stderr: 2022-03-08T18:40:36.556+0000 7f291cd7d700 -1 failed for service _ceph-mon._tcp 2022-03-08T18:40:36.615 INFO:tasks.rook.operator.smithi002.stdout: stderr: 2022-03-08T18:40:36.556+0000 7f291cd7d700 -1 monclient: get_monmap_and_config cannot identify monitors to contact 2022-03-08T18:40:36.615 INFO:tasks.rook.operator.smithi002.stdout: stderr: [errno 2] RADOS object not found (error connecting to the cluster) 2022-03-08T18:40:36.615 INFO:tasks.rook.operator.smithi002.stdout:Traceback (most recent call last): 2022-03-08T18:40:36.615 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/sbin/ceph-volume", line 11, in 2022-03-08T18:40:36.616 INFO:tasks.rook.operator.smithi002.stdout: load_entry_point('ceph-volume==1.0.0', 'console_scripts', 'ceph-volume')() 2022-03-08T18:40:36.616 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 41, in __init__ 2022-03-08T18:40:36.616 INFO:tasks.rook.operator.smithi002.stdout: self.main(self.argv) 2022-03-08T18:40:36.616 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 59, in newfunc 2022-03-08T18:40:36.617 INFO:tasks.rook.operator.smithi002.stdout: return f(*a, **kw) 2022-03-08T18:40:36.617 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 153, in main 2022-03-08T18:40:36.617 INFO:tasks.rook.operator.smithi002.stdout: terminal.dispatch(self.mapper, subcommand_args) 2022-03-08T18:40:36.617 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch 2022-03-08T18:40:36.617 INFO:tasks.rook.operator.smithi002.stdout: instance.main() 2022-03-08T18:40:36.618 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/main.py", line 32, in main 2022-03-08T18:40:36.618 INFO:tasks.rook.operator.smithi002.stdout: terminal.dispatch(self.mapper, self.argv) 2022-03-08T18:40:36.618 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch 2022-03-08T18:40:36.618 INFO:tasks.rook.operator.smithi002.stdout: instance.main() 2022-03-08T18:40:36.619 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 169, in main 2022-03-08T18:40:36.619 INFO:tasks.rook.operator.smithi002.stdout: self.safe_prepare(self.args) 2022-03-08T18:40:36.619 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 91, in safe_prepare 2022-03-08T18:40:36.619 INFO:tasks.rook.operator.smithi002.stdout: self.prepare() 2022-03-08T18:40:36.620 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 16, in is_root 2022-03-08T18:40:36.620 INFO:tasks.rook.operator.smithi002.stdout: return func(*a, **kw) 2022-03-08T18:40:36.620 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 125, in prepare 2022-03-08T18:40:36.620 INFO:tasks.rook.operator.smithi002.stdout: osd_fsid, json.dumps(secrets)) 2022-03-08T18:40:36.621 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/util/prepare.py", line 173, in create_id 2022-03-08T18:40:36.621 INFO:tasks.rook.operator.smithi002.stdout: raise RuntimeError('Unable to create a new OSD id') 2022-03-08T18:40:36.621 INFO:tasks.rook.operator.smithi002.stdout:RuntimeError: Unable to create a new OSD id. debug logs below: 2022-03-08T18:40:36.621 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,102][ceph_volume.main][INFO ] Running command: ceph-volume --log-path /var/log/ceph/smithi002-lv-4-data-0pb5hz raw prepare --bluestore --data /dev/mapper/vg_nvme-lv_4 2022-03-08T18:40:36.621 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,103][ceph_volume.process][INFO ] Running command: /usr/bin/lsblk -plno KNAME,NAME,TYPE 2022-03-08T18:40:36.622 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/loop0 /dev/loop0 loop 2022-03-08T18:40:36.622 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/loop1 /dev/loop1 loop 2022-03-08T18:40:36.622 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/loop2 /dev/loop2 loop 2022-03-08T18:40:36.622 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/loop3 /dev/loop3 loop 2022-03-08T18:40:36.623 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/sda /dev/sda disk 2022-03-08T18:40:36.623 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/sda1 /dev/sda1 part 2022-03-08T18:40:36.623 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-0 /dev/mapper/vg_nvme-lv_1 lvm 2022-03-08T18:40:36.623 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-1 /dev/mapper/vg_nvme-lv_2 lvm 2022-03-08T18:40:36.624 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-2 /dev/mapper/vg_nvme-lv_3 lvm 2022-03-08T18:40:36.624 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-3 /dev/mapper/vg_nvme-lv_4 lvm 2022-03-08T18:40:36.624 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-4 /dev/mapper/vg_nvme-lv_5 lvm 2022-03-08T18:40:36.624 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme0n1 /dev/nvme0n1 disk 2022-03-08T18:40:36.624 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme1n1 /dev/nvme1n1 disk 2022-03-08T18:40:36.625 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme2n1 /dev/nvme2n1 disk 2022-03-08T18:40:36.625 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme3n1 /dev/nvme3n1 disk 2022-03-08T18:40:36.625 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme4n1 /dev/nvme4n1 disk 2022-03-08T18:40:36.625 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,119][ceph_volume.util.system][WARNING] Executable lvs not found on the host, will return lvs as-is 2022-03-08T18:40:36.626 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,119][ceph_volume.process][INFO ] Running command: lvs --noheadings --readonly --separator=";" -a --units=b --nosuffix -S lv_path=/dev/mapper/vg_nvme-lv_4 -o lv_tags,lv_path,lv_name,vg_name,lv_uuid,lv_size 2022-03-08T18:40:36.626 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,406][ceph_volume.process][INFO ] Running command: /usr/bin/lsblk --nodeps -P -o NAME,KNAME,MAJ:MIN,FSTYPE,MOUNTPOINT,LABEL,UUID,RO,RM,MODEL,SIZE,STATE,OWNER,GROUP,MODE,ALIGNMENT,PHY-SEC,LOG-SEC,ROTA,SCHED,TYPE,DISC-ALN,DISC-GRAN,DISC-MAX,DISC-ZERO,PKNAME,PARTLABEL /dev/mapper/vg_nvme-lv_4 2022-03-08T18:40:36.626 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,411][ceph_volume.process][INFO ] stdout NAME="vg_nvme-lv_4" KNAME="dm-3" MAJ:MIN="253:3" FSTYPE="" MOUNTPOINT="" LABEL="" UUID="" RO="0" RM="0" MODEL="" SIZE="89.4G" STATE="running" OWNER="root" GROUP="disk" MODE="brw-rw----" ALIGNMENT="0" PHY-SEC="512" LOG-SEC="512" ROTA="0" SCHED="" TYPE="lvm" DISC-ALN="0" DISC-GRAN="512B" DISC-MAX="2T" DISC-ZERO="0" PKNAME="" PARTLABEL="" 2022-03-08T18:40:36.626 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,412][ceph_volume.process][INFO ] Running command: /usr/sbin/blkid -c /dev/null -p /dev/mapper/vg_nvme-lv_4 2022-03-08T18:40:36.627 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,415][ceph_volume.util.disk][INFO ] opening device /dev/mapper/vg_nvme-lv_4 to check for BlueStore label 2022-03-08T18:40:36.627 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,416][ceph_volume.util.disk][INFO ] opening device /dev/mapper/vg_nvme-lv_4 to check for BlueStore label 2022-03-08T18:40:36.627 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,416][ceph_volume.process][INFO ] Running command: /usr/sbin/udevadm info --query=property /dev/mapper/vg_nvme-lv_4 2022-03-08T18:40:36.627 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DEVLINKS=/dev/disk/by-id/dm-uuid-LVM-bmnmULJa21iqX6XIsAWfS7KzacSWu2ZMJrom0eIpuefSPgiFpeoAc3dLD0UiL91p /dev/vg_nvme/lv_4 /dev/mapper/vg_nvme-lv_4 /dev/disk/by-id/dm-name-vg_nvme-lv_4 2022-03-08T18:40:36.628 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DEVNAME=/dev/dm-3 2022-03-08T18:40:36.628 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DEVPATH=/devices/virtual/block/dm-3 2022-03-08T18:40:36.628 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DEVTYPE=disk 2022-03-08T18:40:36.628 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_LV_NAME=lv_4 2022-03-08T18:40:36.629 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_NAME=vg_nvme-lv_4 2022-03-08T18:40:36.629 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_SUSPENDED=0 2022-03-08T18:40:36.629 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_UDEV_DISABLE_LIBRARY_FALLBACK_FLAG=1 2022-03-08T18:40:36.629 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_UDEV_PRIMARY_SOURCE_FLAG=1 2022-03-08T18:40:36.630 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout DM_UDEV_RULES=1 2022-03-08T18:40:36.630 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout DM_UDEV_RULES_VSN=2 2022-03-08T18:40:36.630 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout DM_UUID=LVM-bmnmULJa21iqX6XIsAWfS7KzacSWu2ZMJrom0eIpuefSPgiFpeoAc3dLD0UiL91p 2022-03-08T18:40:36.630 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout DM_VG_NAME=vg_nvme 2022-03-08T18:40:36.630 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout MAJOR=253 2022-03-08T18:40:36.631 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout MINOR=3 2022-03-08T18:40:36.631 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout SUBSYSTEM=block 2022-03-08T18:40:36.631 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout TAGS=:systemd: 2022-03-08T18:40:36.631 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout USEC_INITIALIZED=231601183 2022-03-08T18:40:36.632 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] Running command: /usr/bin/ceph-authtool --gen-print-key 2022-03-08T18:40:36.632 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,442][ceph_volume.process][INFO ] stdout AQAkoydiec9IGhAAwcAhrWCt9ZbkUL+Er6h81w== 2022-03-08T18:40:36.632 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,443][ceph_volume.process][INFO ] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 43fe8d67-6a36-4902-ad46-836d9bdfb8b1 2022-03-08T18:40:36.632 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,565][ceph_volume.process][INFO ] stderr unable to get monitor info from DNS SRV with service name: ceph-mon 2022-03-08T18:40:36.633 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,566][ceph_volume.process][INFO ] stderr 2022-03-08T18:40:36.556+0000 7f291cd7d700 -1 failed for service _ceph-mon._tcp 2022-03-08T18:40:36.633 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,566][ceph_volume.process][INFO ] stderr 2022-03-08T18:40:36.556+0000 7f291cd7d700 -1 monclient: get_monmap_and_config cannot identify monitors to contact 2022-03-08T18:40:36.633 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,566][ceph_volume.process][INFO ] stderr [errno 2] RADOS object not found (error connecting to the cluster) 2022-03-08T18:40:36.633 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,566][ceph_volume.devices.raw.prepare][ERROR ] raw prepare was unable to complete 2022-03-08T18:40:36.634 INFO:tasks.rook.operator.smithi002.stdout:Traceback (most recent call last): 2022-03-08T18:40:36.634 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 91, in safe_prepare 2022-03-08T18:40:36.634 INFO:tasks.rook.operator.smithi002.stdout: self.prepare() 2022-03-08T18:40:36.634 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 16, in is_root 2022-03-08T18:40:36.634 INFO:tasks.rook.operator.smithi002.stdout: return func(*a, **kw) 2022-03-08T18:40:36.635 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 125, in prepare 2022-03-08T18:40:36.635 INFO:tasks.rook.operator.smithi002.stdout: osd_fsid, json.dumps(secrets)) 2022-03-08T18:40:36.635 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/util/prepare.py", line 173, in create_id 2022-03-08T18:40:36.635 INFO:tasks.rook.operator.smithi002.stdout: raise RuntimeError('Unable to create a new OSD id') 2022-03-08T18:40:36.636 INFO:tasks.rook.operator.smithi002.stdout:RuntimeError: Unable to create a new OSD id 2022-03-08T18:40:36.636 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,568][ceph_volume.devices.raw.prepare][INFO ] will rollback OSD ID creation 2022-03-08T18:40:36.636 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,568][ceph_volume][ERROR ] exception caught by decorator 2022-03-08T18:40:36.636 INFO:tasks.rook.operator.smithi002.stdout:Traceback (most recent call last): 2022-03-08T18:40:36.637 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 59, in newfunc 2022-03-08T18:40:36.637 INFO:tasks.rook.operator.smithi002.stdout: return f(*a, **kw) 2022-03-08T18:40:36.637 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 153, in main 2022-03-08T18:40:36.637 INFO:tasks.rook.operator.smithi002.stdout: terminal.dispatch(self.mapper, subcommand_args) 2022-03-08T18:40:36.638 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch 2022-03-08T18:40:36.638 INFO:tasks.rook.operator.smithi002.stdout: instance.main() 2022-03-08T18:40:36.638 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/main.py", line 32, in main 2022-03-08T18:40:36.638 INFO:tasks.rook.operator.smithi002.stdout: terminal.dispatch(self.mapper, self.argv) 2022-03-08T18:40:36.638 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch 2022-03-08T18:40:36.639 INFO:tasks.rook.operator.smithi002.stdout: instance.main() 2022-03-08T18:40:36.639 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 169, in main 2022-03-08T18:40:36.639 INFO:tasks.rook.operator.smithi002.stdout: self.safe_prepare(self.args) 2022-03-08T18:40:36.639 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 91, in safe_prepare 2022-03-08T18:40:36.640 INFO:tasks.rook.operator.smithi002.stdout: self.prepare() 2022-03-08T18:40:36.640 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 16, in is_root 2022-03-08T18:40:36.640 INFO:tasks.rook.operator.smithi002.stdout: return func(*a, **kw) 2022-03-08T18:40:36.641 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 125, in prepare 2022-03-08T18:40:36.641 INFO:tasks.rook.operator.smithi002.stdout: osd_fsid, json.dumps(secrets)) 2022-03-08T18:40:36.641 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/util/prepare.py", line 173, in create_id 2022-03-08T18:40:36.641 INFO:tasks.rook.operator.smithi002.stdout: raise RuntimeError('Unable to create a new OSD id') 2022-03-08T18:40:36.642 INFO:tasks.rook.operator.smithi002.stdout:RuntimeError: Unable to create a new OSD id 2022-03-08T18:40:36.642 INFO:tasks.rook.operator.smithi002.stdout:: exit status 1} 2022-03-08T18:40:36.653 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:36.654925 I | op-osd: OSD orchestration status for PVC smithi137-lv-2-data-0whzhj is "orchestrating" 2022-03-08T18:40:37.200 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:37.204387 I | op-osd: OSD orchestration status for PVC smithi066-lv-1-data-0drn25 is "completed" 2022-03-08T18:40:37.201 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:37.204407 I | op-osd: creating OSD 0 on PVC "smithi066-lv-1-data-0drn25" 2022-03-08T18:40:37.201 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:37.204414 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-1-data-0drn25" 2022-03-08T18:40:37.830 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:37.834556 I | op-osd: OSD orchestration status for node smithi137-lv-3-data-0kg6xk is "orchestrating" 2022-03-08T18:40:38.691 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:38.694765 I | op-osd: OSD orchestration status for PVC smithi137-lv-3-data-0kg6xk is "orchestrating" 2022-03-08T18:40:39.316 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:39.320042 I | op-osd: OSD orchestration status for node smithi137-lv-4-data-0zchnd is "orchestrating" 2022-03-08T18:40:39.565 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:39.569908 I | op-osd: OSD orchestration status for PVC smithi137-lv-1-data-0cmhmp is "completed" 2022-03-08T18:40:39.566 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:39.569943 I | op-osd: creating OSD 1 on PVC "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:40:39.566 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:39.569958 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:40:39.751 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:39.755255 I | op-osd: OSD orchestration status for node smithi002-lv-4-data-0pb5hz is "orchestrating" 2022-03-08T18:40:39.776 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:39.780032 I | op-osd: OSD orchestration status for PVC smithi002-lv-1-data-0cg8pd is "completed" 2022-03-08T18:40:39.776 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:39.780054 I | op-osd: creating OSD 2 on PVC "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:40:39.777 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:39.780060 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:40:40.852 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:40.855676 I | op-osd: OSD orchestration status for PVC smithi137-lv-4-data-0zchnd is "orchestrating" 2022-03-08T18:40:40.854 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:40.856000 I | op-osd: OSD orchestration status for PVC smithi002-lv-4-data-0pb5hz is "orchestrating" 2022-03-08T18:40:41.129 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:41.133783 E | ceph-crashcollector-controller: node reconcile failed on op "unchanged": Operation cannot be fulfilled on deployments.apps "rook-ceph-crashcollector-smithi002": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T18:40:41.211 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:41.215105 I | op-osd: OSD orchestration status for PVC smithi066-lv-3-data-0xl876 is "completed" 2022-03-08T18:40:41.211 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:41.215140 I | op-osd: creating OSD 3 on PVC "smithi066-lv-3-data-0xl876" 2022-03-08T18:40:41.212 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:41.215160 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-3-data-0xl876" 2022-03-08T18:40:41.640 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:41.644621 E | ceph-crashcollector-controller: node reconcile failed on op "unchanged": Operation cannot be fulfilled on deployments.apps "rook-ceph-crashcollector-smithi002": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T18:40:42.300 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:42.303985 I | op-osd: OSD orchestration status for PVC smithi066-lv-2-data-0jsgjv is "completed" 2022-03-08T18:40:42.300 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:42.304010 I | op-osd: creating OSD 4 on PVC "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:40:42.300 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:42.304020 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:40:43.121 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:40:43.539 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:43.542742 I | op-osd: OSD orchestration status for PVC smithi066-lv-4-data-04lwmk is "completed" 2022-03-08T18:40:43.539 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:43.542822 I | op-osd: creating OSD 6 on PVC "smithi066-lv-4-data-04lwmk" 2022-03-08T18:40:43.540 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:43.542841 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-4-data-04lwmk" 2022-03-08T18:40:44.157 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:44.161037 I | op-osd: OSD orchestration status for PVC smithi137-lv-2-data-0whzhj is "completed" 2022-03-08T18:40:44.157 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:44.161064 I | op-osd: creating OSD 7 on PVC "smithi137-lv-2-data-0whzhj" 2022-03-08T18:40:44.157 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:44.161077 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-2-data-0whzhj" 2022-03-08T18:40:44.195 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:40:44.211 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":17,"num_osds":12,"num_up_osds":0,"osd_up_since":0,"num_in_osds":12,"osd_in_since":1646764842,"num_remapped_pgs":0} 2022-03-08T18:40:44.212 INFO:tasks.rook: have 0/12 OSDs 2022-03-08T18:40:45.717 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:45.717806 I | op-osd: OSD orchestration status for PVC smithi002-lv-2-data-09sjq5 is "completed" 2022-03-08T18:40:45.717 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:45.717840 I | op-osd: creating OSD 5 on PVC "smithi002-lv-2-data-09sjq5" 2022-03-08T18:40:45.717 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:45.717854 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-2-data-09sjq5" 2022-03-08T18:40:46.793 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:46.797423 I | op-osd: OSD orchestration status for PVC smithi002-lv-3-data-05dmhp is "completed" 2022-03-08T18:40:46.795 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:46.797461 I | op-osd: creating OSD 8 on PVC "smithi002-lv-3-data-05dmhp" 2022-03-08T18:40:46.795 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:46.797476 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-3-data-05dmhp" 2022-03-08T18:40:47.327 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:47.331382 I | op-osd: OSD orchestration status for PVC smithi137-lv-3-data-0kg6xk is "completed" 2022-03-08T18:40:47.328 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:47.331425 I | op-osd: creating OSD 9 on PVC "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:40:47.329 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:47.331449 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:40:48.908 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:48.911123 I | op-osd: OSD orchestration status for PVC smithi137-lv-4-data-0zchnd is "completed" 2022-03-08T18:40:48.908 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:48.911146 I | op-osd: creating OSD 10 on PVC "smithi137-lv-4-data-0zchnd" 2022-03-08T18:40:48.909 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:48.911158 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-4-data-0zchnd" 2022-03-08T18:40:51.861 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:51.857668 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". failed to reconcile cluster "rook-ceph": failed to configure local ceph cluster: failed to create cluster: failed to start ceph osds: 1 failures encountered while running osds on nodes in namespace "rook-ceph". 2022-03-08T18:40:51.862 INFO:tasks.rook.operator.smithi002.stdout:failed to provision OSD(s) on PVC smithi002-lv-4-data-0pb5hz. &{OSDs:[] Status:failed PvcBackedOSD:true Message:failed to configure devices: failed to initialize devices on PVC: failed to run ceph-volume. Running command: /usr/bin/ceph-authtool --gen-print-key 2022-03-08T18:40:51.862 INFO:tasks.rook.operator.smithi002.stdout:Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 43fe8d67-6a36-4902-ad46-836d9bdfb8b1 2022-03-08T18:40:51.862 INFO:tasks.rook.operator.smithi002.stdout: stderr: unable to get monitor info from DNS SRV with service name: ceph-mon 2022-03-08T18:40:51.862 INFO:tasks.rook.operator.smithi002.stdout: stderr: 2022-03-08T18:40:36.556+0000 7f291cd7d700 -1 failed for service _ceph-mon._tcp 2022-03-08T18:40:51.863 INFO:tasks.rook.operator.smithi002.stdout: stderr: 2022-03-08T18:40:36.556+0000 7f291cd7d700 -1 monclient: get_monmap_and_config cannot identify monitors to contact 2022-03-08T18:40:51.863 INFO:tasks.rook.operator.smithi002.stdout: stderr: [errno 2] RADOS object not found (error connecting to the cluster) 2022-03-08T18:40:51.863 INFO:tasks.rook.operator.smithi002.stdout:Traceback (most recent call last): 2022-03-08T18:40:51.863 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/sbin/ceph-volume", line 11, in 2022-03-08T18:40:51.864 INFO:tasks.rook.operator.smithi002.stdout: load_entry_point('ceph-volume==1.0.0', 'console_scripts', 'ceph-volume')() 2022-03-08T18:40:51.864 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 41, in __init__ 2022-03-08T18:40:51.864 INFO:tasks.rook.operator.smithi002.stdout: self.main(self.argv) 2022-03-08T18:40:51.864 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 59, in newfunc 2022-03-08T18:40:51.865 INFO:tasks.rook.operator.smithi002.stdout: return f(*a, **kw) 2022-03-08T18:40:51.865 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 153, in main 2022-03-08T18:40:51.865 INFO:tasks.rook.operator.smithi002.stdout: terminal.dispatch(self.mapper, subcommand_args) 2022-03-08T18:40:51.865 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch 2022-03-08T18:40:51.865 INFO:tasks.rook.operator.smithi002.stdout: instance.main() 2022-03-08T18:40:51.866 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/main.py", line 32, in main 2022-03-08T18:40:51.866 INFO:tasks.rook.operator.smithi002.stdout: terminal.dispatch(self.mapper, self.argv) 2022-03-08T18:40:51.866 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch 2022-03-08T18:40:51.866 INFO:tasks.rook.operator.smithi002.stdout: instance.main() 2022-03-08T18:40:51.867 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 169, in main 2022-03-08T18:40:51.867 INFO:tasks.rook.operator.smithi002.stdout: self.safe_prepare(self.args) 2022-03-08T18:40:51.867 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 91, in safe_prepare 2022-03-08T18:40:51.867 INFO:tasks.rook.operator.smithi002.stdout: self.prepare() 2022-03-08T18:40:51.868 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 16, in is_root 2022-03-08T18:40:51.868 INFO:tasks.rook.operator.smithi002.stdout: return func(*a, **kw) 2022-03-08T18:40:51.868 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 125, in prepare 2022-03-08T18:40:51.868 INFO:tasks.rook.operator.smithi002.stdout: osd_fsid, json.dumps(secrets)) 2022-03-08T18:40:51.868 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/util/prepare.py", line 173, in create_id 2022-03-08T18:40:51.869 INFO:tasks.rook.operator.smithi002.stdout: raise RuntimeError('Unable to create a new OSD id') 2022-03-08T18:40:51.869 INFO:tasks.rook.operator.smithi002.stdout:RuntimeError: Unable to create a new OSD id. debug logs below: 2022-03-08T18:40:51.869 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,102][ceph_volume.main][INFO ] Running command: ceph-volume --log-path /var/log/ceph/smithi002-lv-4-data-0pb5hz raw prepare --bluestore --data /dev/mapper/vg_nvme-lv_4 2022-03-08T18:40:51.869 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,103][ceph_volume.process][INFO ] Running command: /usr/bin/lsblk -plno KNAME,NAME,TYPE 2022-03-08T18:40:51.870 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/loop0 /dev/loop0 loop 2022-03-08T18:40:51.870 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/loop1 /dev/loop1 loop 2022-03-08T18:40:51.870 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/loop2 /dev/loop2 loop 2022-03-08T18:40:51.870 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/loop3 /dev/loop3 loop 2022-03-08T18:40:51.871 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/sda /dev/sda disk 2022-03-08T18:40:51.871 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/sda1 /dev/sda1 part 2022-03-08T18:40:51.871 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-0 /dev/mapper/vg_nvme-lv_1 lvm 2022-03-08T18:40:51.871 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-1 /dev/mapper/vg_nvme-lv_2 lvm 2022-03-08T18:40:51.871 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-2 /dev/mapper/vg_nvme-lv_3 lvm 2022-03-08T18:40:51.872 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-3 /dev/mapper/vg_nvme-lv_4 lvm 2022-03-08T18:40:51.872 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-4 /dev/mapper/vg_nvme-lv_5 lvm 2022-03-08T18:40:51.872 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme0n1 /dev/nvme0n1 disk 2022-03-08T18:40:51.872 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme1n1 /dev/nvme1n1 disk 2022-03-08T18:40:51.873 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme2n1 /dev/nvme2n1 disk 2022-03-08T18:40:51.873 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme3n1 /dev/nvme3n1 disk 2022-03-08T18:40:51.873 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme4n1 /dev/nvme4n1 disk 2022-03-08T18:40:51.873 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,119][ceph_volume.util.system][WARNING] Executable lvs not found on the host, will return lvs as-is 2022-03-08T18:40:51.874 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,119][ceph_volume.process][INFO ] Running command: lvs --noheadings --readonly --separator=";" -a --units=b --nosuffix -S lv_path=/dev/mapper/vg_nvme-lv_4 -o lv_tags,lv_path,lv_name,vg_name,lv_uuid,lv_size 2022-03-08T18:40:51.874 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,406][ceph_volume.process][INFO ] Running command: /usr/bin/lsblk --nodeps -P -o NAME,KNAME,MAJ:MIN,FSTYPE,MOUNTPOINT,LABEL,UUID,RO,RM,MODEL,SIZE,STATE,OWNER,GROUP,MODE,ALIGNMENT,PHY-SEC,LOG-SEC,ROTA,SCHED,TYPE,DISC-ALN,DISC-GRAN,DISC-MAX,DISC-ZERO,PKNAME,PARTLABEL /dev/mapper/vg_nvme-lv_4 2022-03-08T18:40:51.874 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,411][ceph_volume.process][INFO ] stdout NAME="vg_nvme-lv_4" KNAME="dm-3" MAJ:MIN="253:3" FSTYPE="" MOUNTPOINT="" LABEL="" UUID="" RO="0" RM="0" MODEL="" SIZE="89.4G" STATE="running" OWNER="root" GROUP="disk" MODE="brw-rw----" ALIGNMENT="0" PHY-SEC="512" LOG-SEC="512" ROTA="0" SCHED="" TYPE="lvm" DISC-ALN="0" DISC-GRAN="512B" DISC-MAX="2T" DISC-ZERO="0" PKNAME="" PARTLABEL="" 2022-03-08T18:40:51.874 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,412][ceph_volume.process][INFO ] Running command: /usr/sbin/blkid -c /dev/null -p /dev/mapper/vg_nvme-lv_4 2022-03-08T18:40:51.874 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,415][ceph_volume.util.disk][INFO ] opening device /dev/mapper/vg_nvme-lv_4 to check for BlueStore label 2022-03-08T18:40:51.875 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,416][ceph_volume.util.disk][INFO ] opening device /dev/mapper/vg_nvme-lv_4 to check for BlueStore label 2022-03-08T18:40:51.875 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,416][ceph_volume.process][INFO ] Running command: /usr/sbin/udevadm info --query=property /dev/mapper/vg_nvme-lv_4 2022-03-08T18:40:51.875 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DEVLINKS=/dev/disk/by-id/dm-uuid-LVM-bmnmULJa21iqX6XIsAWfS7KzacSWu2ZMJrom0eIpuefSPgiFpeoAc3dLD0UiL91p /dev/vg_nvme/lv_4 /dev/mapper/vg_nvme-lv_4 /dev/disk/by-id/dm-name-vg_nvme-lv_4 2022-03-08T18:40:51.875 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DEVNAME=/dev/dm-3 2022-03-08T18:40:51.876 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DEVPATH=/devices/virtual/block/dm-3 2022-03-08T18:40:51.876 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DEVTYPE=disk 2022-03-08T18:40:51.876 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_LV_NAME=lv_4 2022-03-08T18:40:51.876 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_NAME=vg_nvme-lv_4 2022-03-08T18:40:51.877 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_SUSPENDED=0 2022-03-08T18:40:51.877 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_UDEV_DISABLE_LIBRARY_FALLBACK_FLAG=1 2022-03-08T18:40:51.877 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_UDEV_PRIMARY_SOURCE_FLAG=1 2022-03-08T18:40:51.877 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout DM_UDEV_RULES=1 2022-03-08T18:40:51.877 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout DM_UDEV_RULES_VSN=2 2022-03-08T18:40:51.878 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout DM_UUID=LVM-bmnmULJa21iqX6XIsAWfS7KzacSWu2ZMJrom0eIpuefSPgiFpeoAc3dLD0UiL91p 2022-03-08T18:40:51.878 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout DM_VG_NAME=vg_nvme 2022-03-08T18:40:51.878 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout MAJOR=253 2022-03-08T18:40:51.878 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout MINOR=3 2022-03-08T18:40:51.879 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout SUBSYSTEM=block 2022-03-08T18:40:51.879 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout TAGS=:systemd: 2022-03-08T18:40:51.879 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout USEC_INITIALIZED=231601183 2022-03-08T18:40:51.879 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] Running command: /usr/bin/ceph-authtool --gen-print-key 2022-03-08T18:40:51.879 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,442][ceph_volume.process][INFO ] stdout AQAkoydiec9IGhAAwcAhrWCt9ZbkUL+Er6h81w== 2022-03-08T18:40:51.880 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,443][ceph_volume.process][INFO ] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 43fe8d67-6a36-4902-ad46-836d9bdfb8b1 2022-03-08T18:40:51.880 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,565][ceph_volume.process][INFO ] stderr unable to get monitor info from DNS SRV with service name: ceph-mon 2022-03-08T18:40:51.880 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,566][ceph_volume.process][INFO ] stderr 2022-03-08T18:40:36.556+0000 7f291cd7d700 -1 failed for service _ceph-mon._tcp 2022-03-08T18:40:51.880 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,566][ceph_volume.process][INFO ] stderr 2022-03-08T18:40:36.556+0000 7f291cd7d700 -1 monclient: get_monmap_and_config cannot identify monitors to contact 2022-03-08T18:40:51.881 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,566][ceph_volume.process][INFO ] stderr [errno 2] RADOS object not found (error connecting to the cluster) 2022-03-08T18:40:51.881 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,566][ceph_volume.devices.raw.prepare][ERROR ] raw prepare was unable to complete 2022-03-08T18:40:51.881 INFO:tasks.rook.operator.smithi002.stdout:Traceback (most recent call last): 2022-03-08T18:40:51.881 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 91, in safe_prepare 2022-03-08T18:40:51.882 INFO:tasks.rook.operator.smithi002.stdout: self.prepare() 2022-03-08T18:40:51.882 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 16, in is_root 2022-03-08T18:40:51.882 INFO:tasks.rook.operator.smithi002.stdout: return func(*a, **kw) 2022-03-08T18:40:51.882 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 125, in prepare 2022-03-08T18:40:51.883 INFO:tasks.rook.operator.smithi002.stdout: osd_fsid, json.dumps(secrets)) 2022-03-08T18:40:51.883 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/util/prepare.py", line 173, in create_id 2022-03-08T18:40:51.883 INFO:tasks.rook.operator.smithi002.stdout: raise RuntimeError('Unable to create a new OSD id') 2022-03-08T18:40:51.883 INFO:tasks.rook.operator.smithi002.stdout:RuntimeError: Unable to create a new OSD id 2022-03-08T18:40:51.884 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,568][ceph_volume.devices.raw.prepare][INFO ] will rollback OSD ID creation 2022-03-08T18:40:51.884 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,568][ceph_volume][ERROR ] exception caught by decorator 2022-03-08T18:40:51.884 INFO:tasks.rook.operator.smithi002.stdout:Traceback (most recent call last): 2022-03-08T18:40:51.884 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 59, in newfunc 2022-03-08T18:40:51.885 INFO:tasks.rook.operator.smithi002.stdout: return f(*a, **kw) 2022-03-08T18:40:51.885 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 153, in main 2022-03-08T18:40:51.885 INFO:tasks.rook.operator.smithi002.stdout: terminal.dispatch(self.mapper, subcommand_args) 2022-03-08T18:40:51.885 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch 2022-03-08T18:40:51.886 INFO:tasks.rook.operator.smithi002.stdout: instance.main() 2022-03-08T18:40:51.886 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/main.py", line 32, in main 2022-03-08T18:40:51.886 INFO:tasks.rook.operator.smithi002.stdout: terminal.dispatch(self.mapper, self.argv) 2022-03-08T18:40:51.886 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch 2022-03-08T18:40:51.886 INFO:tasks.rook.operator.smithi002.stdout: instance.main() 2022-03-08T18:40:51.887 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 169, in main 2022-03-08T18:40:51.887 INFO:tasks.rook.operator.smithi002.stdout: self.safe_prepare(self.args) 2022-03-08T18:40:51.887 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 91, in safe_prepare 2022-03-08T18:40:51.887 INFO:tasks.rook.operator.smithi002.stdout: self.prepare() 2022-03-08T18:40:51.888 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 16, in is_root 2022-03-08T18:40:51.894 INFO:tasks.rook.operator.smithi002.stdout: return func(*a, **kw) 2022-03-08T18:40:51.894 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 125, in prepare 2022-03-08T18:40:51.894 INFO:tasks.rook.operator.smithi002.stdout: osd_fsid, json.dumps(secrets)) 2022-03-08T18:40:51.894 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/util/prepare.py", line 173, in create_id 2022-03-08T18:40:51.895 INFO:tasks.rook.operator.smithi002.stdout: raise RuntimeError('Unable to create a new OSD id') 2022-03-08T18:40:51.895 INFO:tasks.rook.operator.smithi002.stdout:RuntimeError: Unable to create a new OSD id 2022-03-08T18:40:51.895 INFO:tasks.rook.operator.smithi002.stdout:: exit status 1} 2022-03-08T18:40:51.895 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:51.857731 I | op-k8sutil: Reporting Event rook-ceph:rook-ceph Warning:ReconcileFailed:failed to reconcile cluster "rook-ceph": failed to configure local ceph cluster: failed to create cluster: failed to start ceph osds: 1 failures encountered while running osds on nodes in namespace "rook-ceph". 2022-03-08T18:40:51.896 INFO:tasks.rook.operator.smithi002.stdout:failed to provision OSD(s) on PVC smithi002-lv-4-data-0pb5hz. &{OSDs:[] Status:failed PvcBackedOSD:true Message:failed to configure devices: failed to initialize devices on PVC: failed to run ceph-volume. Running command: /usr/bin/ceph-authtool --gen-print-key 2022-03-08T18:40:51.896 INFO:tasks.rook.operator.smithi002.stdout:Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 43fe8d67-6a36-4902-ad46-836d9bdfb8b1 2022-03-08T18:40:51.896 INFO:tasks.rook.operator.smithi002.stdout: stderr: unable to get monitor info from DNS SRV with service name: ceph-mon 2022-03-08T18:40:51.896 INFO:tasks.rook.operator.smithi002.stdout: stderr: 2022-03-08T18:40:36.556+0000 7f291cd7d700 -1 failed for service _ceph-mon._tcp 2022-03-08T18:40:51.896 INFO:tasks.rook.operator.smithi002.stdout: stderr: 2022-03-08T18:40:36.556+0000 7f291cd7d700 -1 monclient: get_monmap_and_config cannot identify monitors to contact 2022-03-08T18:40:51.897 INFO:tasks.rook.operator.smithi002.stdout: stderr: [errno 2] RADOS object not found (error connecting to the cluster) 2022-03-08T18:40:51.897 INFO:tasks.rook.operator.smithi002.stdout:Traceback (most recent call last): 2022-03-08T18:40:51.897 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/sbin/ceph-volume", line 11, in 2022-03-08T18:40:51.897 INFO:tasks.rook.operator.smithi002.stdout: load_entry_point('ceph-volume==1.0.0', 'console_scripts', 'ceph-volume')() 2022-03-08T18:40:51.898 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 41, in __init__ 2022-03-08T18:40:51.898 INFO:tasks.rook.operator.smithi002.stdout: self.main(self.argv) 2022-03-08T18:40:51.898 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 59, in newfunc 2022-03-08T18:40:51.898 INFO:tasks.rook.operator.smithi002.stdout: return f(*a, **kw) 2022-03-08T18:40:51.899 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 153, in main 2022-03-08T18:40:51.899 INFO:tasks.rook.operator.smithi002.stdout: terminal.dispatch(self.mapper, subcommand_args) 2022-03-08T18:40:51.899 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch 2022-03-08T18:40:51.899 INFO:tasks.rook.operator.smithi002.stdout: instance.main() 2022-03-08T18:40:51.899 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/main.py", line 32, in main 2022-03-08T18:40:51.900 INFO:tasks.rook.operator.smithi002.stdout: terminal.dispatch(self.mapper, self.argv) 2022-03-08T18:40:51.900 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch 2022-03-08T18:40:51.900 INFO:tasks.rook.operator.smithi002.stdout: instance.main() 2022-03-08T18:40:51.900 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 169, in main 2022-03-08T18:40:51.901 INFO:tasks.rook.operator.smithi002.stdout: self.safe_prepare(self.args) 2022-03-08T18:40:51.901 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 91, in safe_prepare 2022-03-08T18:40:51.901 INFO:tasks.rook.operator.smithi002.stdout: self.prepare() 2022-03-08T18:40:51.901 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 16, in is_root 2022-03-08T18:40:51.901 INFO:tasks.rook.operator.smithi002.stdout: return func(*a, **kw) 2022-03-08T18:40:51.902 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 125, in prepare 2022-03-08T18:40:51.902 INFO:tasks.rook.operator.smithi002.stdout: osd_fsid, json.dumps(secrets)) 2022-03-08T18:40:51.902 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/util/prepare.py", line 173, in create_id 2022-03-08T18:40:51.902 INFO:tasks.rook.operator.smithi002.stdout: raise RuntimeError('Unable to create a new OSD id') 2022-03-08T18:40:51.903 INFO:tasks.rook.operator.smithi002.stdout:RuntimeError: Unable to create a new OSD id. debug logs below: 2022-03-08T18:40:51.903 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,102][ceph_volume.main][INFO ] Running command: ceph-volume --log-path /var/log/ceph/smithi002-lv-4-data-0pb5hz raw prepare --bluestore --data /dev/mapper/vg_nvme-lv_4 2022-03-08T18:40:51.903 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,103][ceph_volume.process][INFO ] Running command: /usr/bin/lsblk -plno KNAME,NAME,TYPE 2022-03-08T18:40:51.903 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/loop0 /dev/loop0 loop 2022-03-08T18:40:51.904 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/loop1 /dev/loop1 loop 2022-03-08T18:40:51.904 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/loop2 /dev/loop2 loop 2022-03-08T18:40:51.904 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/loop3 /dev/loop3 loop 2022-03-08T18:40:51.904 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/sda /dev/sda disk 2022-03-08T18:40:51.905 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/sda1 /dev/sda1 part 2022-03-08T18:40:51.905 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-0 /dev/mapper/vg_nvme-lv_1 lvm 2022-03-08T18:40:51.905 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-1 /dev/mapper/vg_nvme-lv_2 lvm 2022-03-08T18:40:51.905 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-2 /dev/mapper/vg_nvme-lv_3 lvm 2022-03-08T18:40:51.905 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-3 /dev/mapper/vg_nvme-lv_4 lvm 2022-03-08T18:40:51.906 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/dm-4 /dev/mapper/vg_nvme-lv_5 lvm 2022-03-08T18:40:51.906 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme0n1 /dev/nvme0n1 disk 2022-03-08T18:40:51.906 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme1n1 /dev/nvme1n1 disk 2022-03-08T18:40:51.906 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme2n1 /dev/nvme2n1 disk 2022-03-08T18:40:51.907 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme3n1 /dev/nvme3n1 disk 2022-03-08T18:40:51.907 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,109][ceph_volume.process][INFO ] stdout /dev/nvme4n1 /dev/nvme4n1 disk 2022-03-08T18:40:51.907 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,119][ceph_volume.util.system][WARNING] Executable lvs not found on the host, will return lvs as-is 2022-03-08T18:40:51.907 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,119][ceph_volume.process][INFO ] Running command: lvs --noheadings --readonly --separator=";" -a --units=b --nosuffix -S lv_path=/dev/mapper/vg_nvme-lv_4 -o lv_tags,lv_path,lv_name,vg_name,lv_uuid,lv_size 2022-03-08T18:40:51.907 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,406][ceph_volume.process][INFO ] Running command: /usr/bin/lsblk --nodeps -P -o NAME,KNAME,MAJ:MIN,FSTYPE,MOUNTPOINT,LABEL,UUID,RO,RM,MODEL,SIZE,STATE,OWNER,GROUP,MODE,ALIGNMENT,PHY-SEC,LOG-SEC,ROTA,SCHED,TYPE,DISC-ALN,DISC-GRAN,DISC-MAX,DISC-ZERO,PKNAME,PARTLABEL /dev/mapper/vg_nvme-lv_4 2022-03-08T18:40:51.908 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,411][ceph_volume.process][INFO ] stdout NAME="vg_nvme-lv_4" KNAME="dm-3" MAJ:MIN="253:3" FSTYPE="" MOUNTPOINT="" LABEL="" UUID="" RO="0" RM="0" MODEL="" SIZE="89.4G" STATE="running" OWNER="root" GROUP="disk" MODE="brw-rw----" ALIGNMENT="0" PHY-SEC="512" LOG-SEC="512" ROTA="0" SCHED="" TYPE="lvm" DISC-ALN="0" DISC-GRAN="512B" DISC-MAX="2T" DISC-ZERO="0" PKNAME="" PARTLABEL="" 2022-03-08T18:40:51.908 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,412][ceph_volume.process][INFO ] Running command: /usr/sbin/blkid -c /dev/null -p /dev/mapper/vg_nvme-lv_4 2022-03-08T18:40:51.908 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,415][ceph_volume.util.disk][INFO ] opening device /dev/mapper/vg_nvme-lv_4 to check for BlueStore label 2022-03-08T18:40:51.908 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,416][ceph_volume.util.disk][INFO ] opening device /dev/mapper/vg_nvme-lv_4 to check for BlueStore label 2022-03-08T18:40:51.909 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,416][ceph_volume.process][INFO ] Running command: /usr/sbin/udevadm info --query=property /dev/mapper/vg_nvme-lv_4 2022-03-08T18:40:51.909 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DEVLINKS=/dev/disk/by-id/dm-uuid-LVM-bmnmULJa21iqX6XIsAWfS7KzacSWu2ZMJrom0eIpuefSPgiFpeoAc3dLD0UiL91p /dev/vg_nvme/lv_4 /dev/mapper/vg_nvme-lv_4 /dev/disk/by-id/dm-name-vg_nvme-lv_4 2022-03-08T18:40:51.909 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DEVNAME=/dev/dm-3 2022-03-08T18:40:51.909 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DEVPATH=/devices/virtual/block/dm-3 2022-03-08T18:40:51.909 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DEVTYPE=disk 2022-03-08T18:40:51.910 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_LV_NAME=lv_4 2022-03-08T18:40:51.910 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_NAME=vg_nvme-lv_4 2022-03-08T18:40:51.910 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_SUSPENDED=0 2022-03-08T18:40:51.910 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_UDEV_DISABLE_LIBRARY_FALLBACK_FLAG=1 2022-03-08T18:40:51.911 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,422][ceph_volume.process][INFO ] stdout DM_UDEV_PRIMARY_SOURCE_FLAG=1 2022-03-08T18:40:51.911 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout DM_UDEV_RULES=1 2022-03-08T18:40:51.911 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout DM_UDEV_RULES_VSN=2 2022-03-08T18:40:51.911 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout DM_UUID=LVM-bmnmULJa21iqX6XIsAWfS7KzacSWu2ZMJrom0eIpuefSPgiFpeoAc3dLD0UiL91p 2022-03-08T18:40:51.911 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout DM_VG_NAME=vg_nvme 2022-03-08T18:40:51.912 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout MAJOR=253 2022-03-08T18:40:51.912 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout MINOR=3 2022-03-08T18:40:51.912 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout SUBSYSTEM=block 2022-03-08T18:40:51.912 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout TAGS=:systemd: 2022-03-08T18:40:51.913 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] stdout USEC_INITIALIZED=231601183 2022-03-08T18:40:51.913 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,423][ceph_volume.process][INFO ] Running command: /usr/bin/ceph-authtool --gen-print-key 2022-03-08T18:40:51.913 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,442][ceph_volume.process][INFO ] stdout AQAkoydiec9IGhAAwcAhrWCt9ZbkUL+Er6h81w== 2022-03-08T18:40:51.913 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,443][ceph_volume.process][INFO ] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 43fe8d67-6a36-4902-ad46-836d9bdfb8b1 2022-03-08T18:40:51.913 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,565][ceph_volume.process][INFO ] stderr unable to get monitor info from DNS SRV with service name: ceph-mon 2022-03-08T18:40:51.914 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,566][ceph_volume.process][INFO ] stderr 2022-03-08T18:40:36.556+0000 7f291cd7d700 -1 failed for service _ceph-mon._tcp 2022-03-08T18:40:51.914 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,566][ceph_volume.process][INFO ] stderr 2022-03-08T18:40:36.556+0000 7f291cd7d700 -1 monclient: get_monmap_and_config cannot identify monitors to contact 2022-03-08T18:40:51.914 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,566][ceph_volume.process][INFO ] stderr [errno 2] RADOS object not found (error connecting to the cluster) 2022-03-08T18:40:51.914 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,566][ceph_volume.devices.raw.prepare][ERROR ] raw prepare was unable to complete 2022-03-08T18:40:51.915 INFO:tasks.rook.operator.smithi002.stdout:Traceback (most recent call last): 2022-03-08T18:40:51.915 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 91, in safe_prepare 2022-03-08T18:40:51.915 INFO:tasks.rook.operator.smithi002.stdout: self.prepare() 2022-03-08T18:40:51.915 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 16, in is_root 2022-03-08T18:40:51.916 INFO:tasks.rook.operator.smithi002.stdout: return func(*a, **kw) 2022-03-08T18:40:51.916 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 125, in prepare 2022-03-08T18:40:51.916 INFO:tasks.rook.operator.smithi002.stdout: osd_fsid, json.dumps(secrets)) 2022-03-08T18:40:51.916 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/util/prepare.py", line 173, in create_id 2022-03-08T18:40:51.916 INFO:tasks.rook.operator.smithi002.stdout: raise RuntimeError('Unable to create a new OSD id') 2022-03-08T18:40:51.917 INFO:tasks.rook.operator.smithi002.stdout:RuntimeError: Unable to create a new OSD id 2022-03-08T18:40:51.917 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,568][ceph_volume.devices.raw.prepare][INFO ] will rollback OSD ID creation 2022-03-08T18:40:51.917 INFO:tasks.rook.operator.smithi002.stdout:[2022-03-08 18:40:36,568][ceph_volume][ERROR ] exception caught by decorator 2022-03-08T18:40:51.917 INFO:tasks.rook.operator.smithi002.stdout:Traceback (most recent call last): 2022-03-08T18:40:51.918 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 59, in newfunc 2022-03-08T18:40:51.918 INFO:tasks.rook.operator.smithi002.stdout: return f(*a, **kw) 2022-03-08T18:40:51.918 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 153, in main 2022-03-08T18:40:51.918 INFO:tasks.rook.operator.smithi002.stdout: terminal.dispatch(self.mapper, subcommand_args) 2022-03-08T18:40:51.919 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch 2022-03-08T18:40:51.919 INFO:tasks.rook.operator.smithi002.stdout: instance.main() 2022-03-08T18:40:51.919 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/main.py", line 32, in main 2022-03-08T18:40:51.919 INFO:tasks.rook.operator.smithi002.stdout: terminal.dispatch(self.mapper, self.argv) 2022-03-08T18:40:51.919 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch 2022-03-08T18:40:51.920 INFO:tasks.rook.operator.smithi002.stdout: instance.main() 2022-03-08T18:40:51.920 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 169, in main 2022-03-08T18:40:51.920 INFO:tasks.rook.operator.smithi002.stdout: self.safe_prepare(self.args) 2022-03-08T18:40:51.920 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 91, in safe_prepare 2022-03-08T18:40:51.921 INFO:tasks.rook.operator.smithi002.stdout: self.prepare() 2022-03-08T18:40:51.921 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 16, in is_root 2022-03-08T18:40:51.921 INFO:tasks.rook.operator.smithi002.stdout: return func(*a, **kw) 2022-03-08T18:40:51.921 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/devices/raw/prepare.py", line 125, in prepare 2022-03-08T18:40:51.921 INFO:tasks.rook.operator.smithi002.stdout: osd_fsid, json.dumps(secrets)) 2022-03-08T18:40:51.922 INFO:tasks.rook.operator.smithi002.stdout: File "/usr/lib/python3.6/site-packages/ceph_volume/util/prepare.py", line 173, in create_id 2022-03-08T18:40:51.922 INFO:tasks.rook.operator.smithi002.stdout: raise RuntimeError('Unable to create a new OSD id') 2022-03-08T18:40:51.922 INFO:tasks.rook.operator.smithi002.stdout:RuntimeError: Unable to create a new OSD id 2022-03-08T18:40:51.922 INFO:tasks.rook.operator.smithi002.stdout:: exit status 1} 2022-03-08T18:40:51.923 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:51.857840 I | ceph-cluster-controller: reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:40:52.584 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:52.587275 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:40:53.055 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:40:53.057981 I | ceph-cluster-controller: detecting the ceph image version for image quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae... 2022-03-08T18:40:54.214 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:40:55.984 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:40:56.003 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":17,"num_osds":12,"num_up_osds":0,"osd_up_since":0,"num_in_osds":12,"osd_in_since":1646764842,"num_remapped_pgs":0} 2022-03-08T18:40:56.003 INFO:tasks.rook: have 0/12 OSDs 2022-03-08T18:41:06.004 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:41:08.205 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:41:08.226 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":20,"num_osds":12,"num_up_osds":0,"osd_up_since":0,"num_in_osds":12,"osd_in_since":1646764842,"num_remapped_pgs":0} 2022-03-08T18:41:08.227 INFO:tasks.rook: have 0/12 OSDs 2022-03-08T18:41:12.336 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:12.337762 I | ceph-cluster-controller: detected ceph image version: "17.0.0-10961 quincy" 2022-03-08T18:41:12.337 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:12.337790 I | ceph-cluster-controller: validating ceph version from provided image 2022-03-08T18:41:12.337 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:12.337803 W | ceph-cluster-controller: unsupported ceph version detected: "17.0.0-10961 quincy", pursuing 2022-03-08T18:41:12.807 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:12.807484 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:41:13.648 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:13.648840 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:41:13.692 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:13.693799 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:41:14.386 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:14.387292 I | ceph-cluster-controller: cluster "rook-ceph": version "17.0.0-10961 quincy" detected for image "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae" 2022-03-08T18:41:15.572 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:15.570202 I | op-mon: start running mons 2022-03-08T18:41:16.663 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:16.663686 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:41:17.629 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:17.629988 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:c=10.109.10.123:6789,a=10.101.136.169:6789,b=10.110.64.66:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:41:17.927 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:17.928798 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:41:17.930 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:17.929011 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:41:17.940 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:17.941752 I | op-mon: targeting the mon count 3 2022-03-08T18:41:17.947 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:17.948455 I | op-config: setting "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:41:18.228 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:41:18.304 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:18.305258 I | op-config: successfully set "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:41:18.304 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:18.305280 I | op-config: setting "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:41:18.677 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:18.678244 I | op-config: successfully set "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:41:18.677 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:18.678275 I | op-config: setting "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:41:18.804 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:41:18.827 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":26,"num_osds":12,"num_up_osds":5,"osd_up_since":1646764877,"num_in_osds":12,"osd_in_since":1646764842,"num_remapped_pgs":0} 2022-03-08T18:41:18.828 INFO:tasks.rook: have 5/12 OSDs 2022-03-08T18:41:19.022 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:19.023324 I | op-config: successfully set "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:41:19.022 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:19.023354 I | op-config: setting "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:41:19.399 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:19.400399 I | op-config: successfully set "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:41:19.400 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:19.400448 I | op-config: setting "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:41:20.031 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:20.031931 I | op-config: successfully set "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:41:20.031 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:20.031960 I | op-config: deleting "log file" option from the mon configuration database 2022-03-08T18:41:20.366 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:20.367322 I | op-config: successfully deleted "log file" option from the mon configuration database 2022-03-08T18:41:20.367 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:20.367350 I | op-mon: checking for basic quorum with existing mons 2022-03-08T18:41:21.231 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:21.231915 I | op-mon: mon "c" endpoint is [v2:10.109.10.123:3300,v1:10.109.10.123:6789] 2022-03-08T18:41:22.331 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:22.332393 I | op-mon: mon "a" endpoint is [v2:10.101.136.169:3300,v1:10.101.136.169:6789] 2022-03-08T18:41:22.789 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:22.789715 I | op-mon: mon "b" endpoint is [v2:10.110.64.66:3300,v1:10.110.64.66:6789] 2022-03-08T18:41:22.878 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:22.878386 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:41:22.893 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:22.894442 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:41:22.894 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:22.894683 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:41:22.920 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:22.921882 I | op-mon: deployment for mon rook-ceph-mon-c already exists. updating if needed 2022-03-08T18:41:22.929 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:22.928844 I | op-k8sutil: deployment "rook-ceph-mon-c" did not change, nothing to update 2022-03-08T18:41:22.929 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:22.928868 I | op-mon: waiting for mon quorum with [c a b] 2022-03-08T18:41:23.354 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:23.355249 I | op-mon: mons running: [c a b] 2022-03-08T18:41:23.821 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:23.821898 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:41:23.830 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:23.831023 I | op-mon: deployment for mon rook-ceph-mon-a already exists. updating if needed 2022-03-08T18:41:23.841 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:23.841899 I | op-k8sutil: deployment "rook-ceph-mon-a" did not change, nothing to update 2022-03-08T18:41:23.841 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:23.841941 I | op-mon: waiting for mon quorum with [c a b] 2022-03-08T18:41:23.958 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:23.958558 I | op-mon: mons running: [c a b] 2022-03-08T18:41:24.372 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:24.373198 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:41:24.379 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:24.380085 I | op-mon: deployment for mon rook-ceph-mon-b already exists. updating if needed 2022-03-08T18:41:24.388 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:24.389375 I | op-k8sutil: deployment "rook-ceph-mon-b" did not change, nothing to update 2022-03-08T18:41:24.389 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:24.389401 I | op-mon: waiting for mon quorum with [c a b] 2022-03-08T18:41:24.558 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:24.558882 I | op-mon: mons running: [c a b] 2022-03-08T18:41:24.982 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:24.983428 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:41:24.983 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:24.983458 I | op-mon: mons created: 3 2022-03-08T18:41:25.451 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:25.451953 I | op-mon: waiting for mon quorum with [c a b] 2022-03-08T18:41:25.730 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:25.730363 I | op-mon: mons running: [c a b] 2022-03-08T18:41:26.144 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:26.145337 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:41:26.145 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:26.145393 I | cephclient: getting or creating ceph auth key "client.csi-rbd-provisioner" 2022-03-08T18:41:26.580 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:26.580676 I | cephclient: getting or creating ceph auth key "client.csi-rbd-node" 2022-03-08T18:41:27.003 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:26.996416 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-provisioner" 2022-03-08T18:41:27.407 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:27.408375 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-node" 2022-03-08T18:41:27.872 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:27.873283 I | ceph-csi: created kubernetes csi secrets for cluster "rook-ceph" 2022-03-08T18:41:27.873 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:27.873313 I | cephclient: getting or creating ceph auth key "client.crash" 2022-03-08T18:41:28.294 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:28.294277 I | ceph-crashcollector-controller: created kubernetes crash collector secret for cluster "rook-ceph" 2022-03-08T18:41:28.775 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:28.775821 I | cephclient: successfully enabled msgr2 protocol 2022-03-08T18:41:28.776 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:28.775865 I | cephclient: create rbd-mirror bootstrap peer token "client.rbd-mirror-peer" 2022-03-08T18:41:28.776 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:28.775877 I | cephclient: getting or creating ceph auth key "client.rbd-mirror-peer" 2022-03-08T18:41:28.829 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:41:29.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.193190 I | cephclient: successfully created rbd-mirror bootstrap peer token for cluster "rook-ceph" 2022-03-08T18:41:29.230 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.230983 I | ceph-spec: controller will reconcile resource "cluster-peer-token-rook-ceph" based on patch: [redacted patch details due to potentially sensitive content] 2022-03-08T18:41:29.244 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.244972 I | op-mgr: start running mgr 2022-03-08T18:41:29.244 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.244997 I | cephclient: getting or creating ceph auth key "mgr.a" 2022-03-08T18:41:29.347 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:41:29.364 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":33,"num_osds":12,"num_up_osds":9,"osd_up_since":1646764887,"num_in_osds":12,"osd_in_since":1646764842,"num_remapped_pgs":0} 2022-03-08T18:41:29.365 INFO:tasks.rook: have 9/12 OSDs 2022-03-08T18:41:29.719 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.719421 I | op-mgr: deployment for mgr rook-ceph-mgr-a already exists. updating if needed 2022-03-08T18:41:29.729 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.729649 I | op-k8sutil: deployment "rook-ceph-mgr-a" did not change, nothing to update 2022-03-08T18:41:29.731 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.732251 I | op-mgr: setting services to point to mgr "a" 2022-03-08T18:41:29.838 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.838499 I | op-osd: start running osds in namespace "rook-ceph" 2022-03-08T18:41:29.838 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.838548 I | op-osd: wait timeout for healthy OSDs during upgrade or restart is "10m0s" 2022-03-08T18:41:29.925 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.925361 I | op-osd: start provisioning the OSDs on PVCs, if needed 2022-03-08T18:41:29.939 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939075 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-1" 2022-03-08T18:41:29.939 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939108 I | op-osd: OSD PVC "smithi002-lv-1-data-0cg8pd" already exists 2022-03-08T18:41:29.939 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939118 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-2" 2022-03-08T18:41:29.940 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939129 I | op-osd: OSD PVC "smithi002-lv-2-data-09sjq5" already exists 2022-03-08T18:41:29.940 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939138 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-3" 2022-03-08T18:41:29.940 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939146 I | op-osd: OSD PVC "smithi002-lv-3-data-05dmhp" already exists 2022-03-08T18:41:29.940 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939153 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-4" 2022-03-08T18:41:29.942 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939161 I | op-osd: OSD PVC "smithi002-lv-4-data-0pb5hz" already exists 2022-03-08T18:41:29.942 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939167 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-1" 2022-03-08T18:41:29.943 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939175 I | op-osd: OSD PVC "smithi066-lv-1-data-0drn25" already exists 2022-03-08T18:41:29.943 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939182 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-2" 2022-03-08T18:41:29.943 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939213 I | op-osd: OSD PVC "smithi066-lv-2-data-0jsgjv" already exists 2022-03-08T18:41:29.943 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939219 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-3" 2022-03-08T18:41:29.944 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939227 I | op-osd: OSD PVC "smithi066-lv-3-data-0xl876" already exists 2022-03-08T18:41:29.944 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939240 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-4" 2022-03-08T18:41:29.944 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939248 I | op-osd: OSD PVC "smithi066-lv-4-data-04lwmk" already exists 2022-03-08T18:41:29.944 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939253 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-1" 2022-03-08T18:41:29.945 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939261 I | op-osd: OSD PVC "smithi137-lv-1-data-0cmhmp" already exists 2022-03-08T18:41:29.945 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939272 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-2" 2022-03-08T18:41:29.945 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939280 I | op-osd: OSD PVC "smithi137-lv-2-data-0whzhj" already exists 2022-03-08T18:41:29.945 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939285 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-3" 2022-03-08T18:41:29.945 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939293 I | op-osd: OSD PVC "smithi137-lv-3-data-0kg6xk" already exists 2022-03-08T18:41:29.946 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939301 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-4" 2022-03-08T18:41:29.946 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.939310 I | op-osd: OSD PVC "smithi137-lv-4-data-0zchnd" already exists 2022-03-08T18:41:29.974 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.971149 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:41:29.975 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.971180 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-2-data-09sjq5" 2022-03-08T18:41:29.975 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.971219 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-3-data-05dmhp" 2022-03-08T18:41:29.975 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:29.971227 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:41:30.277 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:30.277501 I | op-k8sutil: Removing previous job rook-ceph-osd-prepare-smithi002-lv-4-data-0pb5hz to start a new one 2022-03-08T18:41:30.368 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:30.368870 I | op-k8sutil: batch job rook-ceph-osd-prepare-smithi002-lv-4-data-0pb5hz still exists 2022-03-08T18:41:31.145 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:31.145612 I | op-mgr: successful modules: mgr module(s) from the spec 2022-03-08T18:41:31.148 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:31.145827 I | op-mgr: successful modules: dashboard 2022-03-08T18:41:31.148 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:31.146489 I | op-mgr: successful modules: prometheus 2022-03-08T18:41:31.195 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:31.195832 I | op-mgr: successful modules: balancer 2022-03-08T18:41:32.373 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.372094 I | op-k8sutil: batch job rook-ceph-osd-prepare-smithi002-lv-4-data-0pb5hz deleted 2022-03-08T18:41:32.380 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.380004 I | op-osd: started OSD provisioning job for PVC "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:41:32.380 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.380030 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-1-data-0drn25" 2022-03-08T18:41:32.380 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.380038 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:41:32.381 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.380044 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-3-data-0xl876" 2022-03-08T18:41:32.381 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.380049 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-4-data-04lwmk" 2022-03-08T18:41:32.381 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.380055 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:41:32.381 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.380061 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-2-data-0whzhj" 2022-03-08T18:41:32.382 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.380067 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:41:32.382 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.380072 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-4-data-0zchnd" 2022-03-08T18:41:32.382 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.380080 I | op-osd: start provisioning the OSDs on nodes, if needed 2022-03-08T18:41:32.383 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.380085 I | op-osd: no nodes are defined for configuring OSDs on raw devices 2022-03-08T18:41:32.409 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.399944 I | op-osd: OSD orchestration status for PVC smithi002-lv-4-data-0pb5hz is "starting" 2022-03-08T18:41:32.595 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:32.595710 I | op-mgr: successful modules: orchestrator modules 2022-03-08T18:41:34.284 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.283757 I | op-osd: updating OSD 0 on PVC "smithi066-lv-1-data-0drn25" 2022-03-08T18:41:34.284 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.283777 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-1-data-0drn25" 2022-03-08T18:41:34.345 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.345193 I | op-osd: updating OSD 1 on PVC "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:41:34.345 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.345232 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:41:34.428 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.428665 I | op-osd: updating OSD 2 on PVC "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:41:34.429 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.428704 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:41:34.478 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.478547 I | op-osd: updating OSD 3 on PVC "smithi066-lv-3-data-0xl876" 2022-03-08T18:41:34.479 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.478587 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-3-data-0xl876" 2022-03-08T18:41:34.529 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.529441 I | op-osd: updating OSD 4 on PVC "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:41:34.533 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.529492 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:41:34.577 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.577939 I | op-osd: updating OSD 5 on PVC "smithi002-lv-2-data-09sjq5" 2022-03-08T18:41:34.578 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.577972 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-2-data-09sjq5" 2022-03-08T18:41:34.879 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.879046 I | op-osd: updating OSD 6 on PVC "smithi066-lv-4-data-04lwmk" 2022-03-08T18:41:34.879 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:34.879083 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-4-data-04lwmk" 2022-03-08T18:41:35.392 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:35.392585 I | op-osd: updating OSD 8 on PVC "smithi002-lv-3-data-05dmhp" 2022-03-08T18:41:35.392 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:35.392607 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-3-data-05dmhp" 2022-03-08T18:41:35.995 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:35.995754 I | op-osd: updating OSD 9 on PVC "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:41:35.996 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:35.995799 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:41:39.354 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:39.354614 I | op-osd: updating OSD 10 on PVC "smithi137-lv-4-data-0zchnd" 2022-03-08T18:41:39.355 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:39.354647 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-4-data-0zchnd" 2022-03-08T18:41:39.366 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:41:39.900 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:41:39.919 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":38,"num_osds":12,"num_up_osds":11,"osd_up_since":1646764892,"num_in_osds":12,"osd_in_since":1646764842,"num_remapped_pgs":0} 2022-03-08T18:41:39.919 INFO:tasks.rook: have 11/12 OSDs 2022-03-08T18:41:41.036 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:41.036277 I | op-osd: updating OSD 7 on PVC "smithi137-lv-2-data-0whzhj" 2022-03-08T18:41:41.036 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:41.036311 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-2-data-0whzhj" 2022-03-08T18:41:41.963 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:41.963879 I | op-osd: OSD orchestration status for node smithi002-lv-4-data-0pb5hz is "orchestrating" 2022-03-08T18:41:41.994 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:41.989443 I | op-osd: OSD orchestration status for PVC smithi002-lv-4-data-0pb5hz is "orchestrating" 2022-03-08T18:41:43.069 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:43.069559 I | op-osd: OSD orchestration status for PVC smithi002-lv-4-data-0pb5hz is "completed" 2022-03-08T18:41:43.070 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:43.069595 I | op-osd: creating OSD 11 on PVC "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:41:43.070 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:43.069609 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:41:44.101 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:44.100855 I | op-osd: finished running OSDs in namespace "rook-ceph" 2022-03-08T18:41:44.101 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:44.100884 I | ceph-cluster-controller: done reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:41:44.116 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:44.115729 I | op-k8sutil: Reporting Event rook-ceph:rook-ceph Normal:ReconcileSucceeded:successfully configured CephCluster "rook-ceph/rook-ceph" 2022-03-08T18:41:44.117 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:44.115948 I | ceph-cluster-controller: reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:41:44.162 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:44.162069 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:41:44.240 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:44.239974 I | ceph-cluster-controller: detecting the ceph image version for image quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae... 2022-03-08T18:41:49.920 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:41:50.582 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:41:50.611 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":38,"num_osds":12,"num_up_osds":11,"osd_up_since":1646764892,"num_in_osds":12,"osd_in_since":1646764842,"num_remapped_pgs":0} 2022-03-08T18:41:50.611 INFO:tasks.rook: have 11/12 OSDs 2022-03-08T18:41:54.604 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:54.603939 I | ceph-cluster-controller: detected ceph image version: "17.0.0-10961 quincy" 2022-03-08T18:41:54.605 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:54.603975 I | ceph-cluster-controller: validating ceph version from provided image 2022-03-08T18:41:54.606 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:54.603995 W | ceph-cluster-controller: unsupported ceph version detected: "17.0.0-10961 quincy", pursuing 2022-03-08T18:41:55.040 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:55.039833 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:41:55.044 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:55.042434 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:41:55.044 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:55.042718 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:41:55.500 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:55.499227 I | ceph-cluster-controller: cluster "rook-ceph": version "17.0.0-10961 quincy" detected for image "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae" 2022-03-08T18:41:56.114 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:56.113279 I | op-mon: start running mons 2022-03-08T18:41:56.326 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:56.325465 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:41:56.729 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:56.728467 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:41:56.834 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:56.833811 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:41:56.834 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:56.834040 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:41:57.030 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:57.029938 I | op-mon: targeting the mon count 3 2022-03-08T18:41:57.037 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:57.037235 I | op-config: setting "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:41:57.766 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:57.765398 I | op-config: successfully set "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:41:57.766 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:57.765435 I | op-config: setting "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:41:58.108 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:58.107243 I | op-config: successfully set "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:41:58.109 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:58.107282 I | op-config: setting "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:41:58.502 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:58.501182 I | op-config: successfully set "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:41:58.503 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:58.501222 I | op-config: setting "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:41:58.829 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:58.828356 I | op-config: successfully set "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:41:58.829 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:58.828384 I | op-config: setting "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:41:59.205 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:59.204677 I | op-config: successfully set "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:41:59.206 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:59.204710 I | op-config: deleting "log file" option from the mon configuration database 2022-03-08T18:41:59.550 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:59.548998 I | op-config: successfully deleted "log file" option from the mon configuration database 2022-03-08T18:41:59.550 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:59.549032 I | op-mon: checking for basic quorum with existing mons 2022-03-08T18:41:59.918 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:41:59.916998 I | op-mon: mon "c" endpoint is [v2:10.109.10.123:3300,v1:10.109.10.123:6789] 2022-03-08T18:42:00.417 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:00.416819 I | op-mon: mon "a" endpoint is [v2:10.101.136.169:3300,v1:10.101.136.169:6789] 2022-03-08T18:42:00.613 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:42:01.054 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:01.052237 I | op-mon: mon "b" endpoint is [v2:10.110.64.66:3300,v1:10.110.64.66:6789] 2022-03-08T18:42:01.108 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:01.107485 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.110.64.66:6789","10.109.10.123:6789","10.101.136.169:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:42:01.154 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:01.153020 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:42:01.157 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:01.153356 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:42:01.401 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:42:01.423 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":38,"num_osds":12,"num_up_osds":11,"osd_up_since":1646764892,"num_in_osds":12,"osd_in_since":1646764842,"num_remapped_pgs":0} 2022-03-08T18:42:01.423 INFO:tasks.rook: have 11/12 OSDs 2022-03-08T18:42:01.723 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:01.721877 I | op-mon: deployment for mon rook-ceph-mon-c already exists. updating if needed 2022-03-08T18:42:01.763 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:01.762210 I | op-k8sutil: deployment "rook-ceph-mon-c" did not change, nothing to update 2022-03-08T18:42:01.763 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:01.762244 I | op-mon: waiting for mon quorum with [c a b] 2022-03-08T18:42:02.217 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:02.215979 I | op-mon: mons running: [c a b] 2022-03-08T18:42:02.631 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:02.630405 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:42:02.636 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:02.636128 I | op-mon: deployment for mon rook-ceph-mon-a already exists. updating if needed 2022-03-08T18:42:02.646 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:02.645197 I | op-k8sutil: deployment "rook-ceph-mon-a" did not change, nothing to update 2022-03-08T18:42:02.646 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:02.645223 I | op-mon: waiting for mon quorum with [c a b] 2022-03-08T18:42:02.766 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:02.765472 I | op-mon: mons running: [c a b] 2022-03-08T18:42:03.185 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:03.184294 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:42:03.190 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:03.190030 I | op-mon: deployment for mon rook-ceph-mon-b already exists. updating if needed 2022-03-08T18:42:03.199 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:03.198344 I | op-k8sutil: deployment "rook-ceph-mon-b" did not change, nothing to update 2022-03-08T18:42:03.199 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:03.198365 I | op-mon: waiting for mon quorum with [c a b] 2022-03-08T18:42:03.364 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:03.363673 I | op-mon: mons running: [c a b] 2022-03-08T18:42:03.810 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:03.808929 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:42:03.810 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:03.808972 I | op-mon: mons created: 3 2022-03-08T18:42:04.210 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:04.208716 I | op-mon: waiting for mon quorum with [c a b] 2022-03-08T18:42:04.251 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:04.249946 I | op-mon: mons running: [c a b] 2022-03-08T18:42:04.659 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:04.657632 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:42:04.659 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:04.657672 I | cephclient: getting or creating ceph auth key "client.csi-rbd-provisioner" 2022-03-08T18:42:05.074 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:05.072757 I | cephclient: getting or creating ceph auth key "client.csi-rbd-node" 2022-03-08T18:42:05.485 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:05.483412 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-provisioner" 2022-03-08T18:42:05.926 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:05.925007 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-node" 2022-03-08T18:42:06.381 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:06.379950 I | ceph-csi: created kubernetes csi secrets for cluster "rook-ceph" 2022-03-08T18:42:06.382 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:06.379978 I | cephclient: getting or creating ceph auth key "client.crash" 2022-03-08T18:42:06.824 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:06.823076 I | ceph-crashcollector-controller: created kubernetes crash collector secret for cluster "rook-ceph" 2022-03-08T18:42:07.266 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:07.264465 I | cephclient: successfully enabled msgr2 protocol 2022-03-08T18:42:07.266 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:07.264497 I | cephclient: create rbd-mirror bootstrap peer token "client.rbd-mirror-peer" 2022-03-08T18:42:07.267 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:07.264504 I | cephclient: getting or creating ceph auth key "client.rbd-mirror-peer" 2022-03-08T18:42:07.688 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:07.686975 I | cephclient: successfully created rbd-mirror bootstrap peer token for cluster "rook-ceph" 2022-03-08T18:42:07.723 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:07.721364 I | op-mgr: start running mgr 2022-03-08T18:42:07.723 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:07.721412 I | cephclient: getting or creating ceph auth key "mgr.a" 2022-03-08T18:42:08.234 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.232933 I | op-mgr: deployment for mgr rook-ceph-mgr-a already exists. updating if needed 2022-03-08T18:42:08.240 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.239119 I | op-k8sutil: deployment "rook-ceph-mgr-a" did not change, nothing to update 2022-03-08T18:42:08.242 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.241682 I | op-mgr: setting services to point to mgr "a" 2022-03-08T18:42:08.460 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.452165 I | op-osd: start running osds in namespace "rook-ceph" 2022-03-08T18:42:08.463 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.452213 I | op-osd: wait timeout for healthy OSDs during upgrade or restart is "10m0s" 2022-03-08T18:42:08.543 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.541270 I | op-osd: start provisioning the OSDs on PVCs, if needed 2022-03-08T18:42:08.548 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546582 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-1" 2022-03-08T18:42:08.548 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546612 I | op-osd: OSD PVC "smithi002-lv-1-data-0cg8pd" already exists 2022-03-08T18:42:08.553 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546624 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-2" 2022-03-08T18:42:08.553 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546635 I | op-osd: OSD PVC "smithi002-lv-2-data-09sjq5" already exists 2022-03-08T18:42:08.554 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546644 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-3" 2022-03-08T18:42:08.554 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546656 I | op-osd: OSD PVC "smithi002-lv-3-data-05dmhp" already exists 2022-03-08T18:42:08.554 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546665 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-4" 2022-03-08T18:42:08.554 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546677 I | op-osd: OSD PVC "smithi002-lv-4-data-0pb5hz" already exists 2022-03-08T18:42:08.555 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546689 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-1" 2022-03-08T18:42:08.555 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546701 I | op-osd: OSD PVC "smithi066-lv-1-data-0drn25" already exists 2022-03-08T18:42:08.555 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546711 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-2" 2022-03-08T18:42:08.555 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546723 I | op-osd: OSD PVC "smithi066-lv-2-data-0jsgjv" already exists 2022-03-08T18:42:08.556 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546732 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-3" 2022-03-08T18:42:08.556 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546742 I | op-osd: OSD PVC "smithi066-lv-3-data-0xl876" already exists 2022-03-08T18:42:08.556 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546752 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-4" 2022-03-08T18:42:08.557 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546763 I | op-osd: OSD PVC "smithi066-lv-4-data-04lwmk" already exists 2022-03-08T18:42:08.557 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546774 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-1" 2022-03-08T18:42:08.557 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546786 I | op-osd: OSD PVC "smithi137-lv-1-data-0cmhmp" already exists 2022-03-08T18:42:08.557 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546801 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-2" 2022-03-08T18:42:08.558 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546814 I | op-osd: OSD PVC "smithi137-lv-2-data-0whzhj" already exists 2022-03-08T18:42:08.558 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546824 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-3" 2022-03-08T18:42:08.558 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546835 I | op-osd: OSD PVC "smithi137-lv-3-data-0kg6xk" already exists 2022-03-08T18:42:08.558 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546844 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-4" 2022-03-08T18:42:08.559 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.546857 I | op-osd: OSD PVC "smithi137-lv-4-data-0zchnd" already exists 2022-03-08T18:42:08.577 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574070 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:42:08.577 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574091 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-2-data-09sjq5" 2022-03-08T18:42:08.577 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574095 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-3-data-05dmhp" 2022-03-08T18:42:08.578 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574098 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:42:08.578 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574101 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-1-data-0drn25" 2022-03-08T18:42:08.578 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574104 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:42:08.578 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574107 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-3-data-0xl876" 2022-03-08T18:42:08.579 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574110 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-4-data-04lwmk" 2022-03-08T18:42:08.579 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574112 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:42:08.579 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574116 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-2-data-0whzhj" 2022-03-08T18:42:08.579 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574118 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:42:08.580 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574121 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-4-data-0zchnd" 2022-03-08T18:42:08.580 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574124 I | op-osd: start provisioning the OSDs on nodes, if needed 2022-03-08T18:42:08.580 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:08.574127 I | op-osd: no nodes are defined for configuring OSDs on raw devices 2022-03-08T18:42:09.782 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:09.780113 I | op-mgr: successful modules: balancer 2022-03-08T18:42:09.819 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:09.817433 I | op-mgr: successful modules: prometheus 2022-03-08T18:42:09.821 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:09.817815 I | op-mgr: successful modules: dashboard 2022-03-08T18:42:09.855 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:09.853220 I | op-mgr: successful modules: mgr module(s) from the spec 2022-03-08T18:42:10.105 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.103425 I | op-osd: updating OSD 0 on PVC "smithi066-lv-1-data-0drn25" 2022-03-08T18:42:10.105 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.103441 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-1-data-0drn25" 2022-03-08T18:42:10.188 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.186989 I | op-osd: updating OSD 1 on PVC "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:42:10.189 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.187024 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:42:10.241 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.239600 I | op-osd: updating OSD 2 on PVC "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:42:10.241 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.239625 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:42:10.330 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.328199 I | op-osd: updating OSD 3 on PVC "smithi066-lv-3-data-0xl876" 2022-03-08T18:42:10.330 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.328223 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-3-data-0xl876" 2022-03-08T18:42:10.382 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.380265 I | op-osd: updating OSD 4 on PVC "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:42:10.382 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.380300 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:42:10.480 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.478889 I | op-osd: updating OSD 5 on PVC "smithi002-lv-2-data-09sjq5" 2022-03-08T18:42:10.481 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.478924 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-2-data-09sjq5" 2022-03-08T18:42:10.707 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.705121 I | op-osd: updating OSD 6 on PVC "smithi066-lv-4-data-04lwmk" 2022-03-08T18:42:10.707 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:10.705159 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-4-data-04lwmk" 2022-03-08T18:42:11.107 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:11.105523 I | op-osd: updating OSD 8 on PVC "smithi002-lv-3-data-05dmhp" 2022-03-08T18:42:11.107 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:11.105559 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-3-data-05dmhp" 2022-03-08T18:42:11.279 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:11.277446 I | op-mgr: successful modules: orchestrator modules 2022-03-08T18:42:11.425 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd stat -f json 2022-03-08T18:42:11.509 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:11.504979 I | op-osd: updating OSD 9 on PVC "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:42:11.509 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:11.505015 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:42:11.905 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:11.903471 I | op-osd: updating OSD 11 on PVC "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:42:11.906 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:11.903501 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:42:12.002 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:42:12.027 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":42,"num_osds":12,"num_up_osds":12,"osd_up_since":1646764929,"num_in_osds":12,"osd_in_since":1646764842,"num_remapped_pgs":0} 2022-03-08T18:42:12.028 INFO:tasks.rook:Distributing ceph config and client.admin keyring 2022-03-08T18:42:12.028 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- cat /etc/ceph/ceph.conf 2022-03-08T18:42:12.238 INFO:teuthology.orchestra.run.smithi002.stdout:[global] 2022-03-08T18:42:12.238 INFO:teuthology.orchestra.run.smithi002.stdout:mon_host = 10.101.136.169:6789,10.110.64.66:6789,10.109.10.123:6789 2022-03-08T18:42:12.239 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:42:12.239 INFO:teuthology.orchestra.run.smithi002.stdout:[client.admin] 2022-03-08T18:42:12.239 INFO:teuthology.orchestra.run.smithi002.stdout:keyring = /etc/ceph/keyring 2022-03-08T18:42:12.242 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- cat /etc/ceph/keyring 2022-03-08T18:42:12.420 INFO:teuthology.orchestra.run.smithi002.stdout:[client.admin] 2022-03-08T18:42:12.420 INFO:teuthology.orchestra.run.smithi002.stdout:key = AQAtoidi2SMTGRAALmIwZtLC6HcLLp/hvnH4Mw== 2022-03-08T18:42:12.425 DEBUG:teuthology.orchestra.run.smithi002:> sudo mkdir -p /etc/ceph 2022-03-08T18:42:12.439 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /etc/ceph 2022-03-08T18:42:12.457 DEBUG:teuthology.orchestra.run.smithi137:> sudo mkdir -p /etc/ceph 2022-03-08T18:42:12.472 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:42:12.472 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd of=/etc/ceph/ceph.conf 2022-03-08T18:42:12.487 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:42:12.487 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd of=/etc/ceph/keyring 2022-03-08T18:42:12.499 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:42:12.499 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/ceph/ceph.conf 2022-03-08T18:42:12.515 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:42:12.515 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/ceph/keyring 2022-03-08T18:42:12.579 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:42:12.580 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd of=/etc/ceph/ceph.conf 2022-03-08T18:42:12.596 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:42:12.596 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd of=/etc/ceph/keyring 2022-03-08T18:42:12.611 INFO:tasks.rook:Setting up client nodes... 2022-03-08T18:42:12.612 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph auth get-or-create client.a mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2022-03-08T18:42:13.356 INFO:teuthology.orchestra.run.smithi002.stdout:[client.a] 2022-03-08T18:42:13.357 INFO:teuthology.orchestra.run.smithi002.stdout: key = AQCFoydinER7EBAAH3tLuCfkP3YIJS5MzENaVA== 2022-03-08T18:42:13.376 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:42:13.376 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd of=/etc/ceph/ceph.client.a.keyring 2022-03-08T18:42:13.377 DEBUG:teuthology.orchestra.run.smithi002:> sudo chmod 0644 /etc/ceph/ceph.client.a.keyring 2022-03-08T18:42:13.402 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph auth get-or-create client.b mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2022-03-08T18:42:14.148 INFO:teuthology.orchestra.run.smithi002.stdout:[client.b] 2022-03-08T18:42:14.149 INFO:teuthology.orchestra.run.smithi002.stdout: key = AQCGoydiK1UxBBAA+vLoxXk9xSlWiDU2E9J5ww== 2022-03-08T18:42:14.172 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:42:14.172 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/ceph/ceph.client.b.keyring 2022-03-08T18:42:14.173 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod 0644 /etc/ceph/ceph.client.b.keyring 2022-03-08T18:42:14.202 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph auth get-or-create client.c mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2022-03-08T18:42:14.899 INFO:teuthology.orchestra.run.smithi002.stdout:[client.c] 2022-03-08T18:42:14.900 INFO:teuthology.orchestra.run.smithi002.stdout: key = AQCGoydiWYi3LhAAKJNtTTsiHmu9iAiuDEEl+w== 2022-03-08T18:42:14.919 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:42:14.919 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd of=/etc/ceph/ceph.client.c.keyring 2022-03-08T18:42:14.920 DEBUG:teuthology.orchestra.run.smithi137:> sudo chmod 0644 /etc/ceph/ceph.client.c.keyring 2022-03-08T18:42:14.940 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd dump --format=json 2022-03-08T18:42:15.462 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:42:15.463 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":42,"fsid":"79b263ca-2b78-42d8-9481-03fbec358871","created":"2022-03-08T18:37:36.929509+0000","modified":"2022-03-08T18:42:10.822208+0000","last_up_change":"2022-03-08T18:42:09.234734+0000","last_in_change":"2022-03-08T18:40:42.944313+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-03-08T18:41:21.163235+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"on","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"31","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"69a537b9-1292-448d-a4e7-aa929ae84443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":21,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.77:6800","nonce":3617503540},{"type":"v1","addr":"10.248.10.77:6801","nonce":3617503540}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.77:6802","nonce":3617503540},{"type":"v1","addr":"10.248.10.77:6803","nonce":3617503540}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.77:6806","nonce":3617503540},{"type":"v1","addr":"10.248.10.77:6807","nonce":3617503540}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.77:6804","nonce":3617503540},{"type":"v1","addr":"10.248.10.77:6805","nonce":3617503540}]},"public_addr":"10.248.10.77:6801/3617503540","cluster_addr":"10.248.10.77:6803/3617503540","heartbeat_back_addr":"10.248.10.77:6807/3617503540","heartbeat_front_addr":"10.248.10.77:6805/3617503540","state":["exists","up"]},{"osd":1,"uuid":"86aedf8d-d814-4c69-adcd-857dcc5426bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":21,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.142:6800","nonce":3352148772},{"type":"v1","addr":"10.248.15.142:6801","nonce":3352148772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.142:6802","nonce":3352148772},{"type":"v1","addr":"10.248.15.142:6803","nonce":3352148772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.142:6806","nonce":3352148772},{"type":"v1","addr":"10.248.15.142:6807","nonce":3352148772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.142:6804","nonce":3352148772},{"type":"v1","addr":"10.248.15.142:6805","nonce":3352148772}]},"public_addr":"10.248.15.142:6801/3352148772","cluster_addr":"10.248.15.142:6803/3352148772","heartbeat_back_addr":"10.248.15.142:6807/3352148772","heartbeat_front_addr":"10.248.15.142:6805/3352148772","state":["exists","up"]},{"osd":2,"uuid":"7c3fdb59-0464-4b94-9d78-2157bb2b551b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.142:6800","nonce":2239088127},{"type":"v1","addr":"10.248.13.142:6801","nonce":2239088127}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.142:6802","nonce":2239088127},{"type":"v1","addr":"10.248.13.142:6803","nonce":2239088127}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.142:6806","nonce":2239088127},{"type":"v1","addr":"10.248.13.142:6807","nonce":2239088127}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.142:6804","nonce":2239088127},{"type":"v1","addr":"10.248.13.142:6805","nonce":2239088127}]},"public_addr":"10.248.13.142:6801/2239088127","cluster_addr":"10.248.13.142:6803/2239088127","heartbeat_back_addr":"10.248.13.142:6807/2239088127","heartbeat_front_addr":"10.248.13.142:6805/2239088127","state":["exists","up"]},{"osd":3,"uuid":"032b31d3-a555-4d8c-87b4-8b4192fa7c15","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.78:6800","nonce":3709055890},{"type":"v1","addr":"10.248.10.78:6801","nonce":3709055890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.78:6802","nonce":3709055890},{"type":"v1","addr":"10.248.10.78:6803","nonce":3709055890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.78:6806","nonce":3709055890},{"type":"v1","addr":"10.248.10.78:6807","nonce":3709055890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.78:6804","nonce":3709055890},{"type":"v1","addr":"10.248.10.78:6805","nonce":3709055890}]},"public_addr":"10.248.10.78:6801/3709055890","cluster_addr":"10.248.10.78:6803/3709055890","heartbeat_back_addr":"10.248.10.78:6807/3709055890","heartbeat_front_addr":"10.248.10.78:6805/3709055890","state":["exists","up"]},{"osd":4,"uuid":"2a02edfd-991b-4cb8-af09-3852bccfb557","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.79:6800","nonce":2696259370},{"type":"v1","addr":"10.248.10.79:6801","nonce":2696259370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.79:6802","nonce":2696259370},{"type":"v1","addr":"10.248.10.79:6803","nonce":2696259370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.79:6806","nonce":2696259370},{"type":"v1","addr":"10.248.10.79:6807","nonce":2696259370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.79:6804","nonce":2696259370},{"type":"v1","addr":"10.248.10.79:6805","nonce":2696259370}]},"public_addr":"10.248.10.79:6801/2696259370","cluster_addr":"10.248.10.79:6803/2696259370","heartbeat_back_addr":"10.248.10.79:6807/2696259370","heartbeat_front_addr":"10.248.10.79:6805/2696259370","state":["exists","up"]},{"osd":5,"uuid":"f9e41ecb-0de5-4077-bd9a-466a2bcea142","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.143:6800","nonce":3355821867},{"type":"v1","addr":"10.248.13.143:6801","nonce":3355821867}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.143:6802","nonce":3355821867},{"type":"v1","addr":"10.248.13.143:6803","nonce":3355821867}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.143:6806","nonce":3355821867},{"type":"v1","addr":"10.248.13.143:6807","nonce":3355821867}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.143:6804","nonce":3355821867},{"type":"v1","addr":"10.248.13.143:6805","nonce":3355821867}]},"public_addr":"10.248.13.143:6801/3355821867","cluster_addr":"10.248.13.143:6803/3355821867","heartbeat_back_addr":"10.248.13.143:6807/3355821867","heartbeat_front_addr":"10.248.13.143:6805/3355821867","state":["exists","up"]},{"osd":6,"uuid":"bc2210ed-af59-485a-b877-92ee303faa8c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.80:6800","nonce":2255055919},{"type":"v1","addr":"10.248.10.80:6801","nonce":2255055919}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.80:6802","nonce":2255055919},{"type":"v1","addr":"10.248.10.80:6803","nonce":2255055919}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.80:6806","nonce":2255055919},{"type":"v1","addr":"10.248.10.80:6807","nonce":2255055919}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.80:6804","nonce":2255055919},{"type":"v1","addr":"10.248.10.80:6805","nonce":2255055919}]},"public_addr":"10.248.10.80:6801/2255055919","cluster_addr":"10.248.10.80:6803/2255055919","heartbeat_back_addr":"10.248.10.80:6807/2255055919","heartbeat_front_addr":"10.248.10.80:6805/2255055919","state":["exists","up"]},{"osd":7,"uuid":"542412d9-f22e-4f1d-988b-0d0a9c323f2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":29,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.143:6800","nonce":1412058498},{"type":"v1","addr":"10.248.15.143:6801","nonce":1412058498}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.143:6802","nonce":1412058498},{"type":"v1","addr":"10.248.15.143:6803","nonce":1412058498}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.143:6806","nonce":1412058498},{"type":"v1","addr":"10.248.15.143:6807","nonce":1412058498}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.143:6804","nonce":1412058498},{"type":"v1","addr":"10.248.15.143:6805","nonce":1412058498}]},"public_addr":"10.248.15.143:6801/1412058498","cluster_addr":"10.248.15.143:6803/1412058498","heartbeat_back_addr":"10.248.15.143:6807/1412058498","heartbeat_front_addr":"10.248.15.143:6805/1412058498","state":["exists","up"]},{"osd":8,"uuid":"7d5686cd-b26e-4b04-8470-478e32b66fe3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.144:6800","nonce":1031049581},{"type":"v1","addr":"10.248.13.144:6801","nonce":1031049581}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.144:6802","nonce":1031049581},{"type":"v1","addr":"10.248.13.144:6803","nonce":1031049581}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.144:6806","nonce":1031049581},{"type":"v1","addr":"10.248.13.144:6807","nonce":1031049581}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.144:6804","nonce":1031049581},{"type":"v1","addr":"10.248.13.144:6805","nonce":1031049581}]},"public_addr":"10.248.13.144:6801/1031049581","cluster_addr":"10.248.13.144:6803/1031049581","heartbeat_back_addr":"10.248.13.144:6807/1031049581","heartbeat_front_addr":"10.248.13.144:6805/1031049581","state":["exists","up"]},{"osd":9,"uuid":"149581ae-cf54-4b7d-9f91-7d5d4fabf17c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":29,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.144:6800","nonce":378342861},{"type":"v1","addr":"10.248.15.144:6801","nonce":378342861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.144:6802","nonce":378342861},{"type":"v1","addr":"10.248.15.144:6803","nonce":378342861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.144:6806","nonce":378342861},{"type":"v1","addr":"10.248.15.144:6807","nonce":378342861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.144:6804","nonce":378342861},{"type":"v1","addr":"10.248.15.144:6805","nonce":378342861}]},"public_addr":"10.248.15.144:6801/378342861","cluster_addr":"10.248.15.144:6803/378342861","heartbeat_back_addr":"10.248.15.144:6807/378342861","heartbeat_front_addr":"10.248.15.144:6805/378342861","state":["exists","up"]},{"osd":10,"uuid":"293af94b-042d-4bb4-9c77-92f5894c8e58","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":28,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.145:6800","nonce":1178568081},{"type":"v1","addr":"10.248.15.145:6801","nonce":1178568081}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.145:6802","nonce":1178568081},{"type":"v1","addr":"10.248.15.145:6803","nonce":1178568081}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.145:6806","nonce":1178568081},{"type":"v1","addr":"10.248.15.145:6807","nonce":1178568081}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.145:6804","nonce":1178568081},{"type":"v1","addr":"10.248.15.145:6805","nonce":1178568081}]},"public_addr":"10.248.15.145:6801/1178568081","cluster_addr":"10.248.15.145:6803/1178568081","heartbeat_back_addr":"10.248.15.145:6807/1178568081","heartbeat_front_addr":"10.248.15.145:6805/1178568081","state":["exists","up"]},{"osd":11,"uuid":"789ef22f-b5a9-4c5a-b8c6-7a796228a7cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":41,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.147:6800","nonce":3641548723},{"type":"v1","addr":"10.248.13.147:6801","nonce":3641548723}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.147:6802","nonce":3641548723},{"type":"v1","addr":"10.248.13.147:6803","nonce":3641548723}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.147:6806","nonce":3641548723},{"type":"v1","addr":"10.248.13.147:6807","nonce":3641548723}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.147:6804","nonce":3641548723},{"type":"v1","addr":"10.248.13.147:6805","nonce":3641548723}]},"public_addr":"10.248.13.147:6801/3641548723","cluster_addr":"10.248.13.147:6803/3641548723","heartbeat_back_addr":"10.248.13.147:6807/3641548723","heartbeat_front_addr":"10.248.13.147:6805/3641548723","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:02.123622+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:04.775854+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:25.219361+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:11.259426+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:12.140072+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:27.324235+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:15.034098+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:19.153670+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:31.343227+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:19.260505+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:17.972826+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:42:07.193185+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-03-08T18:42:15.484 INFO:tasks.rook.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2022-03-08T18:41:21.163235+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'on', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '31', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2022-03-08T18:42:15.484 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd pool get .mgr pg_num 2022-03-08T18:42:16.071 INFO:teuthology.orchestra.run.smithi002.stdout:pg_num: 1 2022-03-08T18:42:16.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:16.085938 I | op-osd: updating OSD 10 on PVC "smithi137-lv-4-data-0zchnd" 2022-03-08T18:42:16.088 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:16.085972 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-4-data-0zchnd" 2022-03-08T18:42:16.093 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2022-03-08T18:42:16.094 INFO:tasks.rook.ceph_manager.ceph:waiting for mgr available 2022-03-08T18:42:16.094 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph mgr dump --format=json 2022-03-08T18:42:16.652 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:42:16.670 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":16,"active_gid":24242,"active_name":"a","active_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.132:6800","nonce":354781594},{"type":"v1","addr":"10.248.13.132:6801","nonce":354781594}]},"active_addr":"10.248.13.132:6801/354781594","active_change":"2022-03-08T18:39:27.578269+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[],"modules":["iostat","nfs","prometheus","restful","rook"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"0.0.0.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"0.0.0.0","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"prometheus":"http://10.248.13.132:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":4,"active_clients":[{"addrvec":[{"type":"v2","addr":"10.248.13.132:0","nonce":2104135057}]},{"addrvec":[{"type":"v2","addr":"10.248.13.132:0","nonce":2843927063}]},{"addrvec":[{"type":"v2","addr":"10.248.13.132:0","nonce":2957429919}]},{"addrvec":[{"type":"v2","addr":"10.248.13.132:0","nonce":2948140785}]}]}} 2022-03-08T18:42:16.677 INFO:tasks.rook.ceph_manager.ceph:mgr available! 2022-03-08T18:42:16.677 INFO:tasks.rook.ceph_manager.ceph:waiting for all up 2022-03-08T18:42:16.678 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd dump --format=json 2022-03-08T18:42:17.193 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:42:17.194 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":42,"fsid":"79b263ca-2b78-42d8-9481-03fbec358871","created":"2022-03-08T18:37:36.929509+0000","modified":"2022-03-08T18:42:10.822208+0000","last_up_change":"2022-03-08T18:42:09.234734+0000","last_in_change":"2022-03-08T18:40:42.944313+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-03-08T18:41:21.163235+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"on","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"31","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"69a537b9-1292-448d-a4e7-aa929ae84443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":21,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.77:6800","nonce":3617503540},{"type":"v1","addr":"10.248.10.77:6801","nonce":3617503540}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.77:6802","nonce":3617503540},{"type":"v1","addr":"10.248.10.77:6803","nonce":3617503540}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.77:6806","nonce":3617503540},{"type":"v1","addr":"10.248.10.77:6807","nonce":3617503540}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.77:6804","nonce":3617503540},{"type":"v1","addr":"10.248.10.77:6805","nonce":3617503540}]},"public_addr":"10.248.10.77:6801/3617503540","cluster_addr":"10.248.10.77:6803/3617503540","heartbeat_back_addr":"10.248.10.77:6807/3617503540","heartbeat_front_addr":"10.248.10.77:6805/3617503540","state":["exists","up"]},{"osd":1,"uuid":"86aedf8d-d814-4c69-adcd-857dcc5426bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":21,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.142:6800","nonce":3352148772},{"type":"v1","addr":"10.248.15.142:6801","nonce":3352148772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.142:6802","nonce":3352148772},{"type":"v1","addr":"10.248.15.142:6803","nonce":3352148772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.142:6806","nonce":3352148772},{"type":"v1","addr":"10.248.15.142:6807","nonce":3352148772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.142:6804","nonce":3352148772},{"type":"v1","addr":"10.248.15.142:6805","nonce":3352148772}]},"public_addr":"10.248.15.142:6801/3352148772","cluster_addr":"10.248.15.142:6803/3352148772","heartbeat_back_addr":"10.248.15.142:6807/3352148772","heartbeat_front_addr":"10.248.15.142:6805/3352148772","state":["exists","up"]},{"osd":2,"uuid":"7c3fdb59-0464-4b94-9d78-2157bb2b551b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.142:6800","nonce":2239088127},{"type":"v1","addr":"10.248.13.142:6801","nonce":2239088127}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.142:6802","nonce":2239088127},{"type":"v1","addr":"10.248.13.142:6803","nonce":2239088127}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.142:6806","nonce":2239088127},{"type":"v1","addr":"10.248.13.142:6807","nonce":2239088127}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.142:6804","nonce":2239088127},{"type":"v1","addr":"10.248.13.142:6805","nonce":2239088127}]},"public_addr":"10.248.13.142:6801/2239088127","cluster_addr":"10.248.13.142:6803/2239088127","heartbeat_back_addr":"10.248.13.142:6807/2239088127","heartbeat_front_addr":"10.248.13.142:6805/2239088127","state":["exists","up"]},{"osd":3,"uuid":"032b31d3-a555-4d8c-87b4-8b4192fa7c15","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.78:6800","nonce":3709055890},{"type":"v1","addr":"10.248.10.78:6801","nonce":3709055890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.78:6802","nonce":3709055890},{"type":"v1","addr":"10.248.10.78:6803","nonce":3709055890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.78:6806","nonce":3709055890},{"type":"v1","addr":"10.248.10.78:6807","nonce":3709055890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.78:6804","nonce":3709055890},{"type":"v1","addr":"10.248.10.78:6805","nonce":3709055890}]},"public_addr":"10.248.10.78:6801/3709055890","cluster_addr":"10.248.10.78:6803/3709055890","heartbeat_back_addr":"10.248.10.78:6807/3709055890","heartbeat_front_addr":"10.248.10.78:6805/3709055890","state":["exists","up"]},{"osd":4,"uuid":"2a02edfd-991b-4cb8-af09-3852bccfb557","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.79:6800","nonce":2696259370},{"type":"v1","addr":"10.248.10.79:6801","nonce":2696259370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.79:6802","nonce":2696259370},{"type":"v1","addr":"10.248.10.79:6803","nonce":2696259370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.79:6806","nonce":2696259370},{"type":"v1","addr":"10.248.10.79:6807","nonce":2696259370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.79:6804","nonce":2696259370},{"type":"v1","addr":"10.248.10.79:6805","nonce":2696259370}]},"public_addr":"10.248.10.79:6801/2696259370","cluster_addr":"10.248.10.79:6803/2696259370","heartbeat_back_addr":"10.248.10.79:6807/2696259370","heartbeat_front_addr":"10.248.10.79:6805/2696259370","state":["exists","up"]},{"osd":5,"uuid":"f9e41ecb-0de5-4077-bd9a-466a2bcea142","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.143:6800","nonce":3355821867},{"type":"v1","addr":"10.248.13.143:6801","nonce":3355821867}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.143:6802","nonce":3355821867},{"type":"v1","addr":"10.248.13.143:6803","nonce":3355821867}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.143:6806","nonce":3355821867},{"type":"v1","addr":"10.248.13.143:6807","nonce":3355821867}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.143:6804","nonce":3355821867},{"type":"v1","addr":"10.248.13.143:6805","nonce":3355821867}]},"public_addr":"10.248.13.143:6801/3355821867","cluster_addr":"10.248.13.143:6803/3355821867","heartbeat_back_addr":"10.248.13.143:6807/3355821867","heartbeat_front_addr":"10.248.13.143:6805/3355821867","state":["exists","up"]},{"osd":6,"uuid":"bc2210ed-af59-485a-b877-92ee303faa8c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.80:6800","nonce":2255055919},{"type":"v1","addr":"10.248.10.80:6801","nonce":2255055919}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.80:6802","nonce":2255055919},{"type":"v1","addr":"10.248.10.80:6803","nonce":2255055919}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.80:6806","nonce":2255055919},{"type":"v1","addr":"10.248.10.80:6807","nonce":2255055919}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.80:6804","nonce":2255055919},{"type":"v1","addr":"10.248.10.80:6805","nonce":2255055919}]},"public_addr":"10.248.10.80:6801/2255055919","cluster_addr":"10.248.10.80:6803/2255055919","heartbeat_back_addr":"10.248.10.80:6807/2255055919","heartbeat_front_addr":"10.248.10.80:6805/2255055919","state":["exists","up"]},{"osd":7,"uuid":"542412d9-f22e-4f1d-988b-0d0a9c323f2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":29,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.143:6800","nonce":1412058498},{"type":"v1","addr":"10.248.15.143:6801","nonce":1412058498}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.143:6802","nonce":1412058498},{"type":"v1","addr":"10.248.15.143:6803","nonce":1412058498}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.143:6806","nonce":1412058498},{"type":"v1","addr":"10.248.15.143:6807","nonce":1412058498}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.143:6804","nonce":1412058498},{"type":"v1","addr":"10.248.15.143:6805","nonce":1412058498}]},"public_addr":"10.248.15.143:6801/1412058498","cluster_addr":"10.248.15.143:6803/1412058498","heartbeat_back_addr":"10.248.15.143:6807/1412058498","heartbeat_front_addr":"10.248.15.143:6805/1412058498","state":["exists","up"]},{"osd":8,"uuid":"7d5686cd-b26e-4b04-8470-478e32b66fe3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.144:6800","nonce":1031049581},{"type":"v1","addr":"10.248.13.144:6801","nonce":1031049581}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.144:6802","nonce":1031049581},{"type":"v1","addr":"10.248.13.144:6803","nonce":1031049581}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.144:6806","nonce":1031049581},{"type":"v1","addr":"10.248.13.144:6807","nonce":1031049581}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.144:6804","nonce":1031049581},{"type":"v1","addr":"10.248.13.144:6805","nonce":1031049581}]},"public_addr":"10.248.13.144:6801/1031049581","cluster_addr":"10.248.13.144:6803/1031049581","heartbeat_back_addr":"10.248.13.144:6807/1031049581","heartbeat_front_addr":"10.248.13.144:6805/1031049581","state":["exists","up"]},{"osd":9,"uuid":"149581ae-cf54-4b7d-9f91-7d5d4fabf17c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":29,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.144:6800","nonce":378342861},{"type":"v1","addr":"10.248.15.144:6801","nonce":378342861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.144:6802","nonce":378342861},{"type":"v1","addr":"10.248.15.144:6803","nonce":378342861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.144:6806","nonce":378342861},{"type":"v1","addr":"10.248.15.144:6807","nonce":378342861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.144:6804","nonce":378342861},{"type":"v1","addr":"10.248.15.144:6805","nonce":378342861}]},"public_addr":"10.248.15.144:6801/378342861","cluster_addr":"10.248.15.144:6803/378342861","heartbeat_back_addr":"10.248.15.144:6807/378342861","heartbeat_front_addr":"10.248.15.144:6805/378342861","state":["exists","up"]},{"osd":10,"uuid":"293af94b-042d-4bb4-9c77-92f5894c8e58","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":28,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.145:6800","nonce":1178568081},{"type":"v1","addr":"10.248.15.145:6801","nonce":1178568081}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.145:6802","nonce":1178568081},{"type":"v1","addr":"10.248.15.145:6803","nonce":1178568081}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.145:6806","nonce":1178568081},{"type":"v1","addr":"10.248.15.145:6807","nonce":1178568081}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.145:6804","nonce":1178568081},{"type":"v1","addr":"10.248.15.145:6805","nonce":1178568081}]},"public_addr":"10.248.15.145:6801/1178568081","cluster_addr":"10.248.15.145:6803/1178568081","heartbeat_back_addr":"10.248.15.145:6807/1178568081","heartbeat_front_addr":"10.248.15.145:6805/1178568081","state":["exists","up"]},{"osd":11,"uuid":"789ef22f-b5a9-4c5a-b8c6-7a796228a7cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":41,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.147:6800","nonce":3641548723},{"type":"v1","addr":"10.248.13.147:6801","nonce":3641548723}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.147:6802","nonce":3641548723},{"type":"v1","addr":"10.248.13.147:6803","nonce":3641548723}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.147:6806","nonce":3641548723},{"type":"v1","addr":"10.248.13.147:6807","nonce":3641548723}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.147:6804","nonce":3641548723},{"type":"v1","addr":"10.248.13.147:6805","nonce":3641548723}]},"public_addr":"10.248.13.147:6801/3641548723","cluster_addr":"10.248.13.147:6803/3641548723","heartbeat_back_addr":"10.248.13.147:6807/3641548723","heartbeat_front_addr":"10.248.13.147:6805/3641548723","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:02.123622+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:04.775854+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:25.219361+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:11.259426+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:12.140072+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:27.324235+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:15.034098+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:19.153670+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:31.343227+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:19.260505+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:17.972826+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:42:07.193185+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-03-08T18:42:17.213 INFO:tasks.rook.ceph_manager.ceph:all up! 2022-03-08T18:42:17.213 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd dump --format=json 2022-03-08T18:42:17.693 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:17.691286 I | op-osd: updating OSD 7 on PVC "smithi137-lv-2-data-0whzhj" 2022-03-08T18:42:17.694 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:17.691312 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-2-data-0whzhj" 2022-03-08T18:42:17.751 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:42:17.751 INFO:teuthology.orchestra.run.smithi002.stdout:{"epoch":42,"fsid":"79b263ca-2b78-42d8-9481-03fbec358871","created":"2022-03-08T18:37:36.929509+0000","modified":"2022-03-08T18:42:10.822208+0000","last_up_change":"2022-03-08T18:42:09.234734+0000","last_in_change":"2022-03-08T18:40:42.944313+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-03-08T18:41:21.163235+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"on","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"31","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"69a537b9-1292-448d-a4e7-aa929ae84443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":21,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.77:6800","nonce":3617503540},{"type":"v1","addr":"10.248.10.77:6801","nonce":3617503540}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.77:6802","nonce":3617503540},{"type":"v1","addr":"10.248.10.77:6803","nonce":3617503540}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.77:6806","nonce":3617503540},{"type":"v1","addr":"10.248.10.77:6807","nonce":3617503540}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.77:6804","nonce":3617503540},{"type":"v1","addr":"10.248.10.77:6805","nonce":3617503540}]},"public_addr":"10.248.10.77:6801/3617503540","cluster_addr":"10.248.10.77:6803/3617503540","heartbeat_back_addr":"10.248.10.77:6807/3617503540","heartbeat_front_addr":"10.248.10.77:6805/3617503540","state":["exists","up"]},{"osd":1,"uuid":"86aedf8d-d814-4c69-adcd-857dcc5426bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":21,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.142:6800","nonce":3352148772},{"type":"v1","addr":"10.248.15.142:6801","nonce":3352148772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.142:6802","nonce":3352148772},{"type":"v1","addr":"10.248.15.142:6803","nonce":3352148772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.142:6806","nonce":3352148772},{"type":"v1","addr":"10.248.15.142:6807","nonce":3352148772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.142:6804","nonce":3352148772},{"type":"v1","addr":"10.248.15.142:6805","nonce":3352148772}]},"public_addr":"10.248.15.142:6801/3352148772","cluster_addr":"10.248.15.142:6803/3352148772","heartbeat_back_addr":"10.248.15.142:6807/3352148772","heartbeat_front_addr":"10.248.15.142:6805/3352148772","state":["exists","up"]},{"osd":2,"uuid":"7c3fdb59-0464-4b94-9d78-2157bb2b551b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.142:6800","nonce":2239088127},{"type":"v1","addr":"10.248.13.142:6801","nonce":2239088127}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.142:6802","nonce":2239088127},{"type":"v1","addr":"10.248.13.142:6803","nonce":2239088127}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.142:6806","nonce":2239088127},{"type":"v1","addr":"10.248.13.142:6807","nonce":2239088127}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.142:6804","nonce":2239088127},{"type":"v1","addr":"10.248.13.142:6805","nonce":2239088127}]},"public_addr":"10.248.13.142:6801/2239088127","cluster_addr":"10.248.13.142:6803/2239088127","heartbeat_back_addr":"10.248.13.142:6807/2239088127","heartbeat_front_addr":"10.248.13.142:6805/2239088127","state":["exists","up"]},{"osd":3,"uuid":"032b31d3-a555-4d8c-87b4-8b4192fa7c15","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.78:6800","nonce":3709055890},{"type":"v1","addr":"10.248.10.78:6801","nonce":3709055890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.78:6802","nonce":3709055890},{"type":"v1","addr":"10.248.10.78:6803","nonce":3709055890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.78:6806","nonce":3709055890},{"type":"v1","addr":"10.248.10.78:6807","nonce":3709055890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.78:6804","nonce":3709055890},{"type":"v1","addr":"10.248.10.78:6805","nonce":3709055890}]},"public_addr":"10.248.10.78:6801/3709055890","cluster_addr":"10.248.10.78:6803/3709055890","heartbeat_back_addr":"10.248.10.78:6807/3709055890","heartbeat_front_addr":"10.248.10.78:6805/3709055890","state":["exists","up"]},{"osd":4,"uuid":"2a02edfd-991b-4cb8-af09-3852bccfb557","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.79:6800","nonce":2696259370},{"type":"v1","addr":"10.248.10.79:6801","nonce":2696259370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.79:6802","nonce":2696259370},{"type":"v1","addr":"10.248.10.79:6803","nonce":2696259370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.79:6806","nonce":2696259370},{"type":"v1","addr":"10.248.10.79:6807","nonce":2696259370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.79:6804","nonce":2696259370},{"type":"v1","addr":"10.248.10.79:6805","nonce":2696259370}]},"public_addr":"10.248.10.79:6801/2696259370","cluster_addr":"10.248.10.79:6803/2696259370","heartbeat_back_addr":"10.248.10.79:6807/2696259370","heartbeat_front_addr":"10.248.10.79:6805/2696259370","state":["exists","up"]},{"osd":5,"uuid":"f9e41ecb-0de5-4077-bd9a-466a2bcea142","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.143:6800","nonce":3355821867},{"type":"v1","addr":"10.248.13.143:6801","nonce":3355821867}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.143:6802","nonce":3355821867},{"type":"v1","addr":"10.248.13.143:6803","nonce":3355821867}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.143:6806","nonce":3355821867},{"type":"v1","addr":"10.248.13.143:6807","nonce":3355821867}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.143:6804","nonce":3355821867},{"type":"v1","addr":"10.248.13.143:6805","nonce":3355821867}]},"public_addr":"10.248.13.143:6801/3355821867","cluster_addr":"10.248.13.143:6803/3355821867","heartbeat_back_addr":"10.248.13.143:6807/3355821867","heartbeat_front_addr":"10.248.13.143:6805/3355821867","state":["exists","up"]},{"osd":6,"uuid":"bc2210ed-af59-485a-b877-92ee303faa8c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.80:6800","nonce":2255055919},{"type":"v1","addr":"10.248.10.80:6801","nonce":2255055919}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.80:6802","nonce":2255055919},{"type":"v1","addr":"10.248.10.80:6803","nonce":2255055919}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.80:6806","nonce":2255055919},{"type":"v1","addr":"10.248.10.80:6807","nonce":2255055919}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.10.80:6804","nonce":2255055919},{"type":"v1","addr":"10.248.10.80:6805","nonce":2255055919}]},"public_addr":"10.248.10.80:6801/2255055919","cluster_addr":"10.248.10.80:6803/2255055919","heartbeat_back_addr":"10.248.10.80:6807/2255055919","heartbeat_front_addr":"10.248.10.80:6805/2255055919","state":["exists","up"]},{"osd":7,"uuid":"542412d9-f22e-4f1d-988b-0d0a9c323f2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":29,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.143:6800","nonce":1412058498},{"type":"v1","addr":"10.248.15.143:6801","nonce":1412058498}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.143:6802","nonce":1412058498},{"type":"v1","addr":"10.248.15.143:6803","nonce":1412058498}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.143:6806","nonce":1412058498},{"type":"v1","addr":"10.248.15.143:6807","nonce":1412058498}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.143:6804","nonce":1412058498},{"type":"v1","addr":"10.248.15.143:6805","nonce":1412058498}]},"public_addr":"10.248.15.143:6801/1412058498","cluster_addr":"10.248.15.143:6803/1412058498","heartbeat_back_addr":"10.248.15.143:6807/1412058498","heartbeat_front_addr":"10.248.15.143:6805/1412058498","state":["exists","up"]},{"osd":8,"uuid":"7d5686cd-b26e-4b04-8470-478e32b66fe3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.144:6800","nonce":1031049581},{"type":"v1","addr":"10.248.13.144:6801","nonce":1031049581}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.144:6802","nonce":1031049581},{"type":"v1","addr":"10.248.13.144:6803","nonce":1031049581}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.144:6806","nonce":1031049581},{"type":"v1","addr":"10.248.13.144:6807","nonce":1031049581}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.144:6804","nonce":1031049581},{"type":"v1","addr":"10.248.13.144:6805","nonce":1031049581}]},"public_addr":"10.248.13.144:6801/1031049581","cluster_addr":"10.248.13.144:6803/1031049581","heartbeat_back_addr":"10.248.13.144:6807/1031049581","heartbeat_front_addr":"10.248.13.144:6805/1031049581","state":["exists","up"]},{"osd":9,"uuid":"149581ae-cf54-4b7d-9f91-7d5d4fabf17c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":29,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.144:6800","nonce":378342861},{"type":"v1","addr":"10.248.15.144:6801","nonce":378342861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.144:6802","nonce":378342861},{"type":"v1","addr":"10.248.15.144:6803","nonce":378342861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.144:6806","nonce":378342861},{"type":"v1","addr":"10.248.15.144:6807","nonce":378342861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.144:6804","nonce":378342861},{"type":"v1","addr":"10.248.15.144:6805","nonce":378342861}]},"public_addr":"10.248.15.144:6801/378342861","cluster_addr":"10.248.15.144:6803/378342861","heartbeat_back_addr":"10.248.15.144:6807/378342861","heartbeat_front_addr":"10.248.15.144:6805/378342861","state":["exists","up"]},{"osd":10,"uuid":"293af94b-042d-4bb4-9c77-92f5894c8e58","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":28,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.145:6800","nonce":1178568081},{"type":"v1","addr":"10.248.15.145:6801","nonce":1178568081}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.145:6802","nonce":1178568081},{"type":"v1","addr":"10.248.15.145:6803","nonce":1178568081}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.145:6806","nonce":1178568081},{"type":"v1","addr":"10.248.15.145:6807","nonce":1178568081}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.15.145:6804","nonce":1178568081},{"type":"v1","addr":"10.248.15.145:6805","nonce":1178568081}]},"public_addr":"10.248.15.145:6801/1178568081","cluster_addr":"10.248.15.145:6803/1178568081","heartbeat_back_addr":"10.248.15.145:6807/1178568081","heartbeat_front_addr":"10.248.15.145:6805/1178568081","state":["exists","up"]},{"osd":11,"uuid":"789ef22f-b5a9-4c5a-b8c6-7a796228a7cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":41,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.147:6800","nonce":3641548723},{"type":"v1","addr":"10.248.13.147:6801","nonce":3641548723}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.147:6802","nonce":3641548723},{"type":"v1","addr":"10.248.13.147:6803","nonce":3641548723}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.147:6806","nonce":3641548723},{"type":"v1","addr":"10.248.13.147:6807","nonce":3641548723}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.248.13.147:6804","nonce":3641548723},{"type":"v1","addr":"10.248.13.147:6805","nonce":3641548723}]},"public_addr":"10.248.13.147:6801/3641548723","cluster_addr":"10.248.13.147:6803/3641548723","heartbeat_back_addr":"10.248.13.147:6807/3641548723","heartbeat_front_addr":"10.248.13.147:6805/3641548723","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:02.123622+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:04.775854+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:25.219361+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:11.259426+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:12.140072+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:27.324235+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:15.034098+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:19.153670+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:31.343227+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:19.260505+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:41:17.972826+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2022-03-08T18:42:07.193185+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-03-08T18:42:17.771 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph tell osd.0 flush_pg_stats 2022-03-08T18:42:17.772 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph tell osd.1 flush_pg_stats 2022-03-08T18:42:17.772 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph tell osd.2 flush_pg_stats 2022-03-08T18:42:17.773 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph tell osd.3 flush_pg_stats 2022-03-08T18:42:17.774 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph tell osd.4 flush_pg_stats 2022-03-08T18:42:17.775 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph tell osd.5 flush_pg_stats 2022-03-08T18:42:17.776 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph tell osd.6 flush_pg_stats 2022-03-08T18:42:17.777 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph tell osd.7 flush_pg_stats 2022-03-08T18:42:17.777 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph tell osd.8 flush_pg_stats 2022-03-08T18:42:17.778 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph tell osd.9 flush_pg_stats 2022-03-08T18:42:17.779 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph tell osd.10 flush_pg_stats 2022-03-08T18:42:17.779 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph tell osd.11 flush_pg_stats 2022-03-08T18:42:18.237 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:18.235671 I | op-osd: finished running OSDs in namespace "rook-ceph" 2022-03-08T18:42:18.238 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:42:18.235687 I | ceph-cluster-controller: done reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:42:18.358 INFO:teuthology.orchestra.run.smithi002.stdout:90194313232 2022-03-08T18:42:18.358 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.1 2022-03-08T18:42:18.740 INFO:teuthology.orchestra.run.smithi002.stdout:107374182416 2022-03-08T18:42:18.740 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.4 2022-03-08T18:42:18.744 INFO:teuthology.orchestra.run.smithi002.stdout:90194313232 2022-03-08T18:42:18.744 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.0 2022-03-08T18:42:18.747 INFO:teuthology.orchestra.run.smithi002.stdout:107374182416 2022-03-08T18:42:18.748 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.3 2022-03-08T18:42:18.751 INFO:teuthology.orchestra.run.smithi002.stdout:124554051598 2022-03-08T18:42:18.752 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.9 2022-03-08T18:42:18.756 INFO:teuthology.orchestra.run.smithi002.stdout:120259084302 2022-03-08T18:42:18.756 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.10 2022-03-08T18:42:18.759 INFO:teuthology.orchestra.run.smithi002.stdout:111669149710 2022-03-08T18:42:18.759 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.6 2022-03-08T18:42:18.762 INFO:teuthology.orchestra.run.smithi002.stdout:176093659141 2022-03-08T18:42:18.763 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.11 2022-03-08T18:42:18.765 INFO:teuthology.orchestra.run.smithi002.stdout:146028888077 2022-03-08T18:42:18.766 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.5 2022-03-08T18:42:18.799 INFO:teuthology.orchestra.run.smithi002.stdout:158913789964 2022-03-08T18:42:18.799 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.8 2022-03-08T18:42:18.823 INFO:teuthology.orchestra.run.smithi002.stdout:124554051597 2022-03-08T18:42:18.824 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.7 2022-03-08T18:42:18.850 INFO:teuthology.orchestra.run.smithi002.stdout:137438953486 2022-03-08T18:42:18.851 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.2 2022-03-08T18:42:19.194 INFO:teuthology.orchestra.run.smithi002.stdout:90194313231 2022-03-08T18:42:19.307 INFO:tasks.rook.ceph_manager.ceph:need seq 90194313232 got 90194313231 for osd.1 2022-03-08T18:42:19.506 INFO:teuthology.orchestra.run.smithi002.stdout:107374182414 2022-03-08T18:42:19.631 INFO:tasks.rook.ceph_manager.ceph:need seq 107374182416 got 107374182414 for osd.4 2022-03-08T18:42:19.631 INFO:teuthology.orchestra.run.smithi002.stdout:111669149710 2022-03-08T18:42:19.645 INFO:teuthology.orchestra.run.smithi002.stdout:120259084302 2022-03-08T18:42:19.719 INFO:teuthology.orchestra.run.smithi002.stdout:176093659141 2022-03-08T18:42:19.728 INFO:tasks.rook.ceph_manager.ceph:need seq 111669149710 got 111669149710 for osd.6 2022-03-08T18:42:19.729 DEBUG:teuthology.parallel:result is None 2022-03-08T18:42:19.731 INFO:tasks.rook.ceph_manager.ceph:need seq 120259084302 got 120259084302 for osd.10 2022-03-08T18:42:19.732 DEBUG:teuthology.parallel:result is None 2022-03-08T18:42:19.763 INFO:tasks.rook.ceph_manager.ceph:need seq 176093659141 got 176093659141 for osd.11 2022-03-08T18:42:19.763 DEBUG:teuthology.parallel:result is None 2022-03-08T18:42:19.819 INFO:teuthology.orchestra.run.smithi002.stdout:90194313232 2022-03-08T18:42:19.831 INFO:teuthology.orchestra.run.smithi002.stdout:107374182416 2022-03-08T18:42:19.852 INFO:tasks.rook.ceph_manager.ceph:need seq 90194313232 got 90194313232 for osd.0 2022-03-08T18:42:19.853 DEBUG:teuthology.parallel:result is None 2022-03-08T18:42:19.866 INFO:tasks.rook.ceph_manager.ceph:need seq 107374182416 got 107374182416 for osd.3 2022-03-08T18:42:19.867 DEBUG:teuthology.parallel:result is None 2022-03-08T18:42:19.885 INFO:teuthology.orchestra.run.smithi002.stdout:124554051598 2022-03-08T18:42:19.904 INFO:tasks.rook.ceph_manager.ceph:need seq 124554051598 got 124554051598 for osd.9 2022-03-08T18:42:19.905 DEBUG:teuthology.parallel:result is None 2022-03-08T18:42:19.999 INFO:teuthology.orchestra.run.smithi002.stdout:146028888076 2022-03-08T18:42:20.023 INFO:tasks.rook.ceph_manager.ceph:need seq 146028888077 got 146028888076 for osd.5 2022-03-08T18:42:20.044 INFO:teuthology.orchestra.run.smithi002.stdout:158913789963 2022-03-08T18:42:20.069 INFO:tasks.rook.ceph_manager.ceph:need seq 158913789964 got 158913789963 for osd.8 2022-03-08T18:42:20.094 INFO:teuthology.orchestra.run.smithi002.stdout:137438953485 2022-03-08T18:42:20.116 INFO:tasks.rook.ceph_manager.ceph:need seq 137438953486 got 137438953485 for osd.2 2022-03-08T18:42:20.127 INFO:teuthology.orchestra.run.smithi002.stdout:124554051596 2022-03-08T18:42:20.146 INFO:tasks.rook.ceph_manager.ceph:need seq 124554051597 got 124554051596 for osd.7 2022-03-08T18:42:20.308 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.1 2022-03-08T18:42:20.632 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.4 2022-03-08T18:42:20.857 INFO:teuthology.orchestra.run.smithi002.stdout:90194313232 2022-03-08T18:42:20.878 INFO:tasks.rook.ceph_manager.ceph:need seq 90194313232 got 90194313232 for osd.1 2022-03-08T18:42:20.878 DEBUG:teuthology.parallel:result is None 2022-03-08T18:42:21.024 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.5 2022-03-08T18:42:21.070 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.8 2022-03-08T18:42:21.117 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.2 2022-03-08T18:42:21.123 INFO:teuthology.orchestra.run.smithi002.stdout:107374182416 2022-03-08T18:42:21.146 INFO:tasks.rook.ceph_manager.ceph:need seq 107374182416 got 107374182416 for osd.4 2022-03-08T18:42:21.146 DEBUG:teuthology.parallel:result is None 2022-03-08T18:42:21.148 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.7 2022-03-08T18:42:21.557 INFO:teuthology.orchestra.run.smithi002.stdout:146028888076 2022-03-08T18:42:21.577 INFO:tasks.rook.ceph_manager.ceph:need seq 146028888077 got 146028888076 for osd.5 2022-03-08T18:42:21.636 INFO:teuthology.orchestra.run.smithi002.stdout:158913789963 2022-03-08T18:42:21.642 INFO:teuthology.orchestra.run.smithi002.stdout:137438953485 2022-03-08T18:42:21.659 INFO:tasks.rook.ceph_manager.ceph:need seq 158913789964 got 158913789963 for osd.8 2022-03-08T18:42:21.662 INFO:tasks.rook.ceph_manager.ceph:need seq 137438953486 got 137438953485 for osd.2 2022-03-08T18:42:21.710 INFO:teuthology.orchestra.run.smithi002.stdout:124554051596 2022-03-08T18:42:21.741 INFO:tasks.rook.ceph_manager.ceph:need seq 124554051597 got 124554051596 for osd.7 2022-03-08T18:42:22.578 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.5 2022-03-08T18:42:22.660 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.8 2022-03-08T18:42:22.662 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.2 2022-03-08T18:42:22.742 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd last-stat-seq osd.7 2022-03-08T18:42:23.146 INFO:teuthology.orchestra.run.smithi002.stdout:146028888078 2022-03-08T18:42:23.167 INFO:tasks.rook.ceph_manager.ceph:need seq 146028888077 got 146028888078 for osd.5 2022-03-08T18:42:23.167 DEBUG:teuthology.parallel:result is None 2022-03-08T18:42:23.204 INFO:teuthology.orchestra.run.smithi002.stdout:158913789965 2022-03-08T18:42:23.223 INFO:tasks.rook.ceph_manager.ceph:need seq 158913789964 got 158913789965 for osd.8 2022-03-08T18:42:23.224 DEBUG:teuthology.parallel:result is None 2022-03-08T18:42:23.319 INFO:teuthology.orchestra.run.smithi002.stdout:137438953486 2022-03-08T18:42:23.338 INFO:tasks.rook.ceph_manager.ceph:need seq 137438953486 got 137438953486 for osd.2 2022-03-08T18:42:23.339 DEBUG:teuthology.parallel:result is None 2022-03-08T18:42:23.371 INFO:teuthology.orchestra.run.smithi002.stdout:124554051598 2022-03-08T18:42:23.390 INFO:tasks.rook.ceph_manager.ceph:need seq 124554051597 got 124554051598 for osd.7 2022-03-08T18:42:23.391 DEBUG:teuthology.parallel:result is None 2022-03-08T18:42:23.391 INFO:tasks.rook.ceph_manager.ceph:waiting for clean 2022-03-08T18:42:23.391 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph pg dump --format=json 2022-03-08T18:42:23.925 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:42:23.928 INFO:teuthology.orchestra.run.smithi002.stderr:dumped all 2022-03-08T18:42:23.946 INFO:teuthology.orchestra.run.smithi002.stdout:{"pg_ready":true,"pg_map":{"version":118,"stamp":"2022-03-08T18:42:22.655308+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":405516,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":12,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":1124941824,"kb_used":67236,"kb_used_data":6404,"kb_used_omap":0,"kb_used_meta":60736,"kb_avail":1124874588,"statfs":{"total":1151940427776,"available":1151871578112,"internally_reserved":0,"allocated":6557696,"data_stored":3652085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":62193664},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.623633"},"pg_stats":[{"pgid":"1.0","version":"38'137","reported_seq":218,"reported_epoch":41,"state":"active+clean","last_fresh":"2022-03-08T18:42:09.440585+0000","last_change":"2022-03-08T18:41:31.860362+0000","last_active":"2022-03-08T18:42:09.440585+0000","last_peered":"2022-03-08T18:42:09.440585+0000","last_clean":"2022-03-08T18:42:09.440585+0000","last_became_active":"2022-03-08T18:41:31.846159+0000","last_became_peered":"2022-03-08T18:41:31.846159+0000","last_unstale":"2022-03-08T18:42:09.440585+0000","last_undegraded":"2022-03-08T18:42:09.440585+0000","last_fullsized":"2022-03-08T18:42:09.440585+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":28,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-03-08T18:41:21.664165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-03-08T18:41:21.664165+0000","last_clean_scrub_stamp":"2022-03-08T18:41:21.664165+0000","objects_scrubbed":0,"log_size":137,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-03-09T22:20:08.737327+0000","stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":405516,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":405516,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2007040,"data_stored":1992300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":11,"up_from":41,"seq":176093659142,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5212,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739940,"statfs":{"total":95995035648,"available":95989698560,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,6,7,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.92700000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.3500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.125}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1040000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.5620000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.2949999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.0680000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.1790000000000003}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.2510000000000003}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.1280000000000001}]}]},{"osd":8,"up_from":37,"seq":158913789965,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5276,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739876,"statfs":{"total":95995035648,"available":95989633024,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,5,6,7,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68799999999999994}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80600000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.107}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0600000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.4399999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.351}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.6829999999999998}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.0939999999999999}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.5590000000000002}]}]},{"osd":5,"up_from":34,"seq":146028888078,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6112,"kb_used_data":920,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739040,"statfs":{"total":95995035648,"available":95988776960,"internally_reserved":0,"allocated":942080,"data_stored":699699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73099999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.492}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.423}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.387}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.5960000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0800000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.663}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.6819999999999999}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.48}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.6140000000000001}]}]},{"osd":2,"up_from":32,"seq":137438953486,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5276,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739876,"statfs":{"total":95995035648,"available":95989633024,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,3,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78500000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.92700000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.304}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.98899999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.753}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.724}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.46}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.5900000000000001}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.6100000000000001}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.7110000000000001}]}]},{"osd":9,"up_from":29,"seq":124554051599,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5340,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739812,"statfs":{"total":95995035648,"available":95989567488,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6,7,8,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.95699999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47699999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1930000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.131}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.099}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.98499999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0329999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.01}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.997}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.93999999999999995}]}]},{"osd":7,"up_from":29,"seq":124554051598,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6436,"kb_used_data":924,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738716,"statfs":{"total":95995035648,"available":95988445184,"internally_reserved":0,"allocated":946176,"data_stored":701158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,4,5,6,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.421}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.341}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.762}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.94399999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.5269999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.8600000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.597}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.575}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.5419999999999998}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.651}]}]},{"osd":1,"up_from":21,"seq":90194313233,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5404,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739748,"statfs":{"total":95995035648,"available":95989501952,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,2,3,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 8 18:42:13 2022","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.175,"15min":1.175},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":2.2589999999999999,"5min":2.2589999999999999,"15min":2.2589999999999999},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.369,"5min":1.369,"15min":1.369},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":2.347,"5min":2.347,"15min":2.347},"last":1.79}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.2290000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.256}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.79800000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.93899999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.3580000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1890000000000001}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.4419999999999999}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.4019999999999999}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.722}]}]},{"osd":0,"up_from":21,"seq":90194313233,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[1,2,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Mar 8 18:42:11 2022","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.109,"15min":1.109},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.8520000000000001,"5min":1.8520000000000001,"15min":1.8520000000000001},"last":1.363},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"min":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":2.3199999999999998,"5min":2.3199999999999998,"15min":2.3199999999999998},"last":1.0649999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.329}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.0369999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.4620000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.4419999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.6150000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.179}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.8340000000000001}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.6080000000000001}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.7729999999999999}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.1699999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182417,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5340,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739812,"statfs":{"total":95995035648,"available":95989567488,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 8 18:42:20 2022","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.115,"15min":1.115},"min":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":2.7810000000000001,"5min":2.7810000000000001,"15min":2.7810000000000001},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.1279999999999999,"15min":1.1279999999999999},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":2.6549999999999998,"5min":2.6549999999999998,"15min":2.6549999999999998},"last":0.91400000000000003}]},{"osd":1,"last update":"Tue Mar 8 18:42:20 2022","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":0.89900000000000002},{"interface":"front","average":{"1min":1.224,"5min":1.224,"15min":1.224},"min":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"max":{"1min":1.802,"5min":1.802,"15min":1.802},"last":1.4950000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.607}]},{"osd":4,"last update":"Tue Mar 8 18:42:20 2022","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":1.6140000000000001,"15min":1.6140000000000001},"min":{"1min":1.044,"5min":1.044,"15min":1.044},"max":{"1min":2.1309999999999998,"5min":2.1309999999999998,"15min":2.1309999999999998},"last":1.2050000000000001},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":3.214,"5min":3.214,"15min":3.214},"last":1.03}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.083}]},{"osd":6,"last update":"Tue Mar 8 18:42:20 2022","interfaces":[{"interface":"back","average":{"1min":1.829,"5min":1.829,"15min":1.829},"min":{"1min":0.879,"5min":0.879,"15min":0.879},"max":{"1min":2.7090000000000001,"5min":2.7090000000000001,"15min":2.7090000000000001},"last":1.7949999999999999},{"interface":"front","average":{"1min":2.0219999999999998,"5min":2.0219999999999998,"15min":2.0219999999999998},"min":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"max":{"1min":2.7890000000000001,"5min":2.7890000000000001,"15min":2.7890000000000001},"last":1.534}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.298}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.3620000000000001}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.9119999999999999}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.0009999999999999}]}]},{"osd":10,"up_from":28,"seq":120259084302,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6436,"kb_used_data":924,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738716,"statfs":{"total":95995035648,"available":95988445184,"internally_reserved":0,"allocated":946176,"data_stored":701158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[1,2,3,4,5,6,7,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.85799999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.92200000000000004}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.5609999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.6100000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.101}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0429999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.131}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0740000000000001}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.754}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.71}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.51}]}]},{"osd":4,"up_from":25,"seq":107374182417,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5340,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739812,"statfs":{"total":95995035648,"available":95989567488,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 8 18:42:19 2022","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.123,"15min":1.123},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.6779999999999999,"5min":1.6779999999999999,"15min":1.6779999999999999},"last":0.96399999999999997},{"interface":"front","average":{"1min":1.2809999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"min":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.998,"5min":1.998,"15min":1.998},"last":1.113}]},{"osd":1,"last update":"Tue Mar 8 18:42:19 2022","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":1.167,"15min":1.167},"min":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"max":{"1min":1.7230000000000001,"5min":1.7230000000000001,"15min":1.7230000000000001},"last":1.1579999999999999},{"interface":"front","average":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"min":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"max":{"1min":1.8480000000000001,"5min":1.8480000000000001,"15min":1.8480000000000001},"last":1.831}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.51}]},{"osd":3,"last update":"Tue Mar 8 18:42:19 2022","interfaces":[{"interface":"back","average":{"1min":1.5580000000000001,"5min":1.5580000000000001,"15min":1.5580000000000001},"min":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"max":{"1min":2.512,"5min":2.512,"15min":2.512},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.5960000000000001,"15min":1.5960000000000001},"min":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"max":{"1min":2.6579999999999999,"5min":2.6579999999999999,"15min":2.6579999999999999},"last":1.609}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.536}]},{"osd":6,"last update":"Tue Mar 8 18:42:19 2022","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":1.7809999999999999,"15min":1.7809999999999999},"min":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":2.4820000000000002,"5min":2.4820000000000002,"15min":2.4820000000000002},"last":1.677},{"interface":"front","average":{"1min":1.9039999999999999,"5min":1.9039999999999999,"15min":1.9039999999999999},"min":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"max":{"1min":2.7679999999999998,"5min":2.7679999999999998,"15min":2.7679999999999998},"last":2.109}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.974}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.909}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.0089999999999999}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.226}]}]},{"osd":6,"up_from":26,"seq":111669149711,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5340,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739812,"statfs":{"total":95995035648,"available":95989567488,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 8 18:42:18 2022","interfaces":[{"interface":"back","average":{"1min":1.375,"5min":1.375,"15min":1.375},"min":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"max":{"1min":3.1659999999999999,"5min":3.1659999999999999,"15min":3.1659999999999999},"last":2.0299999999999998},{"interface":"front","average":{"1min":1.131,"5min":1.131,"15min":1.131},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":2.1659999999999999,"5min":2.1659999999999999,"15min":2.1659999999999999},"last":0.98999999999999999}]},{"osd":1,"last update":"Tue Mar 8 18:42:18 2022","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.234,"15min":1.234},"min":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"max":{"1min":2.0819999999999999,"5min":2.0819999999999999,"15min":2.0819999999999999},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.339,"5min":1.339,"15min":1.339},"min":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"max":{"1min":2.0880000000000001,"5min":2.0880000000000001,"15min":2.0880000000000001},"last":0.95099999999999996}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.228}]},{"osd":3,"last update":"Tue Mar 8 18:42:18 2022","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":1.597,"15min":1.597},"min":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"max":{"1min":2.242,"5min":2.242,"15min":2.242},"last":2.1429999999999998},{"interface":"front","average":{"1min":1.569,"5min":1.569,"15min":1.569},"min":{"1min":0.877,"5min":0.877,"15min":0.877},"max":{"1min":2.1520000000000001,"5min":2.1520000000000001,"15min":2.1520000000000001},"last":1.982}]},{"osd":4,"last update":"Tue Mar 8 18:42:18 2022","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.72,"15min":1.72},"min":{"1min":1.056,"5min":1.056,"15min":1.056},"max":{"1min":3.0030000000000001,"5min":3.0030000000000001,"15min":3.0030000000000001},"last":2.1000000000000001},{"interface":"front","average":{"1min":1.7290000000000001,"5min":1.7290000000000001,"15min":1.7290000000000001},"min":{"1min":1.1220000000000001,"5min":1.1220000000000001,"15min":1.1220000000000001},"max":{"1min":2.4220000000000002,"5min":2.4220000000000002,"15min":2.4220000000000002},"last":1.7989999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.2839999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.2120000000000002}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.4489999999999998}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.2410000000000001}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.3879999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":409600,"data_stored":405516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-03-08T18:42:23.950 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph pg dump --format=json 2022-03-08T18:42:24.514 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:42:24.516 INFO:teuthology.orchestra.run.smithi002.stderr:dumped all 2022-03-08T18:42:24.542 INFO:teuthology.orchestra.run.smithi002.stdout:{"pg_ready":true,"pg_map":{"version":118,"stamp":"2022-03-08T18:42:22.655308+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":405516,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":12,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":1124941824,"kb_used":67236,"kb_used_data":6404,"kb_used_omap":0,"kb_used_meta":60736,"kb_avail":1124874588,"statfs":{"total":1151940427776,"available":1151871578112,"internally_reserved":0,"allocated":6557696,"data_stored":3652085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":62193664},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.623633"},"pg_stats":[{"pgid":"1.0","version":"38'137","reported_seq":218,"reported_epoch":41,"state":"active+clean","last_fresh":"2022-03-08T18:42:09.440585+0000","last_change":"2022-03-08T18:41:31.860362+0000","last_active":"2022-03-08T18:42:09.440585+0000","last_peered":"2022-03-08T18:42:09.440585+0000","last_clean":"2022-03-08T18:42:09.440585+0000","last_became_active":"2022-03-08T18:41:31.846159+0000","last_became_peered":"2022-03-08T18:41:31.846159+0000","last_unstale":"2022-03-08T18:42:09.440585+0000","last_undegraded":"2022-03-08T18:42:09.440585+0000","last_fullsized":"2022-03-08T18:42:09.440585+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":28,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-03-08T18:41:21.664165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-03-08T18:41:21.664165+0000","last_clean_scrub_stamp":"2022-03-08T18:41:21.664165+0000","objects_scrubbed":0,"log_size":137,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2022-03-09T22:20:08.737327+0000","stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":405516,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":405516,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2007040,"data_stored":1992300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":11,"up_from":41,"seq":176093659142,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5212,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739940,"statfs":{"total":95995035648,"available":95989698560,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,6,7,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.92700000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.3500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.125}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1040000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.5620000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.2949999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.0680000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.1790000000000003}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.2510000000000003}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.1280000000000001}]}]},{"osd":8,"up_from":37,"seq":158913789965,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5276,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739876,"statfs":{"total":95995035648,"available":95989633024,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,5,6,7,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68799999999999994}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80600000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.107}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0600000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.4399999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.351}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.6829999999999998}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.0939999999999999}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.5590000000000002}]}]},{"osd":5,"up_from":34,"seq":146028888078,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6112,"kb_used_data":920,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739040,"statfs":{"total":95995035648,"available":95988776960,"internally_reserved":0,"allocated":942080,"data_stored":699699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73099999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.492}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.423}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.387}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.5960000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0800000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.663}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.6819999999999999}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.48}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.6140000000000001}]}]},{"osd":2,"up_from":32,"seq":137438953486,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5276,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739876,"statfs":{"total":95995035648,"available":95989633024,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,3,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78500000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.92700000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.304}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.98899999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.753}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.724}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.46}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.5900000000000001}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.6100000000000001}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.7110000000000001}]}]},{"osd":9,"up_from":29,"seq":124554051599,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5340,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739812,"statfs":{"total":95995035648,"available":95989567488,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6,7,8,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.95699999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47699999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1930000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.131}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.099}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.98499999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0329999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.01}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.997}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.93999999999999995}]}]},{"osd":7,"up_from":29,"seq":124554051598,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6436,"kb_used_data":924,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738716,"statfs":{"total":95995035648,"available":95988445184,"internally_reserved":0,"allocated":946176,"data_stored":701158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,4,5,6,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.421}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.341}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.762}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.94399999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.5269999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.8600000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.597}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.575}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.5419999999999998}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.651}]}]},{"osd":1,"up_from":21,"seq":90194313233,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5404,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739748,"statfs":{"total":95995035648,"available":95989501952,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,2,3,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 8 18:42:13 2022","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.175,"15min":1.175},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":2.2589999999999999,"5min":2.2589999999999999,"15min":2.2589999999999999},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.369,"5min":1.369,"15min":1.369},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":2.347,"5min":2.347,"15min":2.347},"last":1.79}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.2290000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.256}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.79800000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.93899999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.3580000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1890000000000001}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.4419999999999999}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.4019999999999999}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.722}]}]},{"osd":0,"up_from":21,"seq":90194313233,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[1,2,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Mar 8 18:42:11 2022","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.109,"15min":1.109},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.8520000000000001,"5min":1.8520000000000001,"15min":1.8520000000000001},"last":1.363},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"min":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":2.3199999999999998,"5min":2.3199999999999998,"15min":2.3199999999999998},"last":1.0649999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.329}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.0369999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.4620000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.4419999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.6150000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.179}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.8340000000000001}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.6080000000000001}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.7729999999999999}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.1699999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182417,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5340,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739812,"statfs":{"total":95995035648,"available":95989567488,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 8 18:42:20 2022","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.115,"15min":1.115},"min":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":2.7810000000000001,"5min":2.7810000000000001,"15min":2.7810000000000001},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.1279999999999999,"15min":1.1279999999999999},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":2.6549999999999998,"5min":2.6549999999999998,"15min":2.6549999999999998},"last":0.91400000000000003}]},{"osd":1,"last update":"Tue Mar 8 18:42:20 2022","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":0.89900000000000002},{"interface":"front","average":{"1min":1.224,"5min":1.224,"15min":1.224},"min":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"max":{"1min":1.802,"5min":1.802,"15min":1.802},"last":1.4950000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.607}]},{"osd":4,"last update":"Tue Mar 8 18:42:20 2022","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":1.6140000000000001,"15min":1.6140000000000001},"min":{"1min":1.044,"5min":1.044,"15min":1.044},"max":{"1min":2.1309999999999998,"5min":2.1309999999999998,"15min":2.1309999999999998},"last":1.2050000000000001},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":3.214,"5min":3.214,"15min":3.214},"last":1.03}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.083}]},{"osd":6,"last update":"Tue Mar 8 18:42:20 2022","interfaces":[{"interface":"back","average":{"1min":1.829,"5min":1.829,"15min":1.829},"min":{"1min":0.879,"5min":0.879,"15min":0.879},"max":{"1min":2.7090000000000001,"5min":2.7090000000000001,"15min":2.7090000000000001},"last":1.7949999999999999},{"interface":"front","average":{"1min":2.0219999999999998,"5min":2.0219999999999998,"15min":2.0219999999999998},"min":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"max":{"1min":2.7890000000000001,"5min":2.7890000000000001,"15min":2.7890000000000001},"last":1.534}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.298}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.3620000000000001}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.9119999999999999}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.0009999999999999}]}]},{"osd":10,"up_from":28,"seq":120259084302,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6436,"kb_used_data":924,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738716,"statfs":{"total":95995035648,"available":95988445184,"internally_reserved":0,"allocated":946176,"data_stored":701158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[1,2,3,4,5,6,7,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.85799999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.92200000000000004}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.5609999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.6100000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.101}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0429999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.131}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0740000000000001}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.754}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.71}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.51}]}]},{"osd":4,"up_from":25,"seq":107374182417,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5340,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739812,"statfs":{"total":95995035648,"available":95989567488,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 8 18:42:19 2022","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.123,"15min":1.123},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.6779999999999999,"5min":1.6779999999999999,"15min":1.6779999999999999},"last":0.96399999999999997},{"interface":"front","average":{"1min":1.2809999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"min":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.998,"5min":1.998,"15min":1.998},"last":1.113}]},{"osd":1,"last update":"Tue Mar 8 18:42:19 2022","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":1.167,"15min":1.167},"min":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"max":{"1min":1.7230000000000001,"5min":1.7230000000000001,"15min":1.7230000000000001},"last":1.1579999999999999},{"interface":"front","average":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"min":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"max":{"1min":1.8480000000000001,"5min":1.8480000000000001,"15min":1.8480000000000001},"last":1.831}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.51}]},{"osd":3,"last update":"Tue Mar 8 18:42:19 2022","interfaces":[{"interface":"back","average":{"1min":1.5580000000000001,"5min":1.5580000000000001,"15min":1.5580000000000001},"min":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"max":{"1min":2.512,"5min":2.512,"15min":2.512},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.5960000000000001,"15min":1.5960000000000001},"min":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"max":{"1min":2.6579999999999999,"5min":2.6579999999999999,"15min":2.6579999999999999},"last":1.609}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.536}]},{"osd":6,"last update":"Tue Mar 8 18:42:19 2022","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":1.7809999999999999,"15min":1.7809999999999999},"min":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":2.4820000000000002,"5min":2.4820000000000002,"15min":2.4820000000000002},"last":1.677},{"interface":"front","average":{"1min":1.9039999999999999,"5min":1.9039999999999999,"15min":1.9039999999999999},"min":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"max":{"1min":2.7679999999999998,"5min":2.7679999999999998,"15min":2.7679999999999998},"last":2.109}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.974}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.909}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.0089999999999999}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.226}]}]},{"osd":6,"up_from":26,"seq":111669149711,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5340,"kb_used_data":404,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739812,"statfs":{"total":95995035648,"available":95989567488,"internally_reserved":0,"allocated":413696,"data_stored":172230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 8 18:42:18 2022","interfaces":[{"interface":"back","average":{"1min":1.375,"5min":1.375,"15min":1.375},"min":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"max":{"1min":3.1659999999999999,"5min":3.1659999999999999,"15min":3.1659999999999999},"last":2.0299999999999998},{"interface":"front","average":{"1min":1.131,"5min":1.131,"15min":1.131},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":2.1659999999999999,"5min":2.1659999999999999,"15min":2.1659999999999999},"last":0.98999999999999999}]},{"osd":1,"last update":"Tue Mar 8 18:42:18 2022","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.234,"15min":1.234},"min":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"max":{"1min":2.0819999999999999,"5min":2.0819999999999999,"15min":2.0819999999999999},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.339,"5min":1.339,"15min":1.339},"min":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"max":{"1min":2.0880000000000001,"5min":2.0880000000000001,"15min":2.0880000000000001},"last":0.95099999999999996}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.228}]},{"osd":3,"last update":"Tue Mar 8 18:42:18 2022","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":1.597,"15min":1.597},"min":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"max":{"1min":2.242,"5min":2.242,"15min":2.242},"last":2.1429999999999998},{"interface":"front","average":{"1min":1.569,"5min":1.569,"15min":1.569},"min":{"1min":0.877,"5min":0.877,"15min":0.877},"max":{"1min":2.1520000000000001,"5min":2.1520000000000001,"15min":2.1520000000000001},"last":1.982}]},{"osd":4,"last update":"Tue Mar 8 18:42:18 2022","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.72,"15min":1.72},"min":{"1min":1.056,"5min":1.056,"15min":1.056},"max":{"1min":3.0030000000000001,"5min":3.0030000000000001,"15min":3.0030000000000001},"last":2.1000000000000001},{"interface":"front","average":{"1min":1.7290000000000001,"5min":1.7290000000000001,"15min":1.7290000000000001},"min":{"1min":1.1220000000000001,"5min":1.1220000000000001,"15min":1.1220000000000001},"max":{"1min":2.4220000000000002,"5min":2.4220000000000002,"15min":2.4220000000000002},"last":1.7989999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.2839999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.2120000000000002}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.4489999999999998}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.2410000000000001}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.3879999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":409600,"data_stored":405516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-03-08T18:42:24.545 INFO:tasks.rook.ceph_manager.ceph:clean! 2022-03-08T18:42:24.545 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2022-03-08T18:42:24.546 INFO:tasks.rook.ceph_manager.ceph:wait_until_healthy 2022-03-08T18:42:24.546 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph health --format=json 2022-03-08T18:42:25.100 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:42:25.101 INFO:teuthology.orchestra.run.smithi002.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2022-03-08T18:42:25.119 INFO:tasks.rook.ceph_manager.ceph:wait_until_healthy done 2022-03-08T18:42:25.120 INFO:tasks.rook:Rook complete, yielding 2022-03-08T18:42:25.120 INFO:teuthology.run_tasks:Running task rook.shell... 2022-03-08T18:42:25.137 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph -s 2022-03-08T18:42:25.757 INFO:teuthology.orchestra.run.smithi002.stdout: cluster: 2022-03-08T18:42:25.757 INFO:teuthology.orchestra.run.smithi002.stdout: id: 79b263ca-2b78-42d8-9481-03fbec358871 2022-03-08T18:42:25.757 INFO:teuthology.orchestra.run.smithi002.stdout: health: HEALTH_OK 2022-03-08T18:42:25.757 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:42:25.758 INFO:teuthology.orchestra.run.smithi002.stdout: services: 2022-03-08T18:42:25.758 INFO:teuthology.orchestra.run.smithi002.stdout: mon: 3 daemons, quorum a,b,c (age 3m) 2022-03-08T18:42:25.758 INFO:teuthology.orchestra.run.smithi002.stdout: mgr: a(active, since 2m) 2022-03-08T18:42:25.758 INFO:teuthology.orchestra.run.smithi002.stdout: osd: 12 osds: 12 up (since 16s), 12 in (since 102s) 2022-03-08T18:42:25.759 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:42:25.759 INFO:teuthology.orchestra.run.smithi002.stdout: data: 2022-03-08T18:42:25.759 INFO:teuthology.orchestra.run.smithi002.stdout: pools: 1 pools, 1 pgs 2022-03-08T18:42:25.759 INFO:teuthology.orchestra.run.smithi002.stdout: objects: 2 objects, 577 KiB 2022-03-08T18:42:25.759 INFO:teuthology.orchestra.run.smithi002.stdout: usage: 66 MiB used, 1.0 TiB / 1.0 TiB avail 2022-03-08T18:42:25.760 INFO:teuthology.orchestra.run.smithi002.stdout: pgs: 1 active+clean 2022-03-08T18:42:25.760 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:42:25.779 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch status 2022-03-08T18:42:26.516 INFO:teuthology.orchestra.run.smithi002.stdout:Backend: rook 2022-03-08T18:42:26.517 INFO:teuthology.orchestra.run.smithi002.stdout:Available: Yes 2022-03-08T18:42:26.548 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ps 2022-03-08T18:42:27.120 INFO:teuthology.orchestra.run.smithi002.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID 2022-03-08T18:42:27.120 INFO:teuthology.orchestra.run.smithi002.stdout:crashcollector.crash smithi002 running (106s) 0s ago 106s - - 8a96e9d18eed 2022-03-08T18:42:27.120 INFO:teuthology.orchestra.run.smithi002.stdout:crashcollector.crash smithi066 running (3m) 0s ago 3m - - 8a96e9d18eed 2022-03-08T18:42:27.121 INFO:teuthology.orchestra.run.smithi002.stdout:crashcollector.crash smithi137 running (3m) 0s ago 3m - - 8a96e9d18eed 2022-03-08T18:42:27.121 INFO:teuthology.orchestra.run.smithi002.stdout:mgr.a smithi002 running (3m) 0s ago 3m - - 8a96e9d18eed 2022-03-08T18:42:27.121 INFO:teuthology.orchestra.run.smithi002.stdout:mon.a smithi137 running (5m) 0s ago 5m - - 8a96e9d18eed 2022-03-08T18:42:27.121 INFO:teuthology.orchestra.run.smithi002.stdout:mon.b smithi066 running (4m) 0s ago 4m - - 8a96e9d18eed 2022-03-08T18:42:27.121 INFO:teuthology.orchestra.run.smithi002.stdout:mon.c smithi002 running (4m) 0s ago 4m - - 8a96e9d18eed 2022-03-08T18:42:27.122 INFO:teuthology.orchestra.run.smithi002.stdout:osd.0 smithi066 running (110s) 0s ago 110s - - 8a96e9d18eed 2022-03-08T18:42:27.122 INFO:teuthology.orchestra.run.smithi002.stdout:osd.1 smithi137 running (108s) 0s ago 108s - - 8a96e9d18eed 2022-03-08T18:42:27.122 INFO:teuthology.orchestra.run.smithi002.stdout:osd.10 smithi137 running (96s) 0s ago 96s - - 8a96e9d18eed 2022-03-08T18:42:27.122 INFO:teuthology.orchestra.run.smithi002.stdout:osd.11 smithi002 running (44s) 0s ago 44s - - 8a96e9d18eed 2022-03-08T18:42:27.123 INFO:teuthology.orchestra.run.smithi002.stdout:osd.2 smithi002 running (107s) 0s ago 107s - - 8a96e9d18eed 2022-03-08T18:42:27.123 INFO:teuthology.orchestra.run.smithi002.stdout:osd.3 smithi066 running (105s) 0s ago 105s - - 8a96e9d18eed 2022-03-08T18:42:27.123 INFO:teuthology.orchestra.run.smithi002.stdout:osd.4 smithi066 running (104s) 0s ago 104s - - 8a96e9d18eed 2022-03-08T18:42:27.123 INFO:teuthology.orchestra.run.smithi002.stdout:osd.5 smithi002 running (101s) 0s ago 101s - - 8a96e9d18eed 2022-03-08T18:42:27.123 INFO:teuthology.orchestra.run.smithi002.stdout:osd.6 smithi066 running (103s) 0s ago 103s - - 8a96e9d18eed 2022-03-08T18:42:27.124 INFO:teuthology.orchestra.run.smithi002.stdout:osd.7 smithi137 running (101s) 0s ago 102s - - 8a96e9d18eed 2022-03-08T18:42:27.124 INFO:teuthology.orchestra.run.smithi002.stdout:osd.8 smithi002 running (100s) 0s ago 100s - - 8a96e9d18eed 2022-03-08T18:42:27.124 INFO:teuthology.orchestra.run.smithi002.stdout:osd.9 smithi137 running (99s) 0s ago 99s - - 8a96e9d18eed 2022-03-08T18:42:27.140 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls 2022-03-08T18:42:27.849 INFO:teuthology.orchestra.run.smithi002.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2022-03-08T18:42:27.850 INFO:teuthology.orchestra.run.smithi002.stdout:crash 3/3 0s ago 3m * 2022-03-08T18:42:27.850 INFO:teuthology.orchestra.run.smithi002.stdout:mgr 1/1 0s ago 3m count:1 2022-03-08T18:42:27.850 INFO:teuthology.orchestra.run.smithi002.stdout:mon 3/3 0s ago 5m count:3 2022-03-08T18:42:27.851 INFO:teuthology.orchestra.run.smithi002.stdout:osd 24 0s ago 110s 2022-03-08T18:42:27.851 INFO:teuthology.orchestra.run.smithi002.stdout:osd.all-available-devices 0 0s ago - * 2022-03-08T18:42:27.875 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch device ls 2022-03-08T18:42:28.426 INFO:teuthology.orchestra.run.smithi002.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REFRESHED REJECT REASONS 2022-03-08T18:42:28.427 INFO:teuthology.orchestra.run.smithi002.stdout:smithi002 /dev/vg_nvme/lv_1 unknown None 107G 0s ago 2022-03-08T18:42:28.427 INFO:teuthology.orchestra.run.smithi002.stdout:smithi002 /dev/vg_nvme/lv_2 unknown None 107G 0s ago 2022-03-08T18:42:28.428 INFO:teuthology.orchestra.run.smithi002.stdout:smithi002 /dev/vg_nvme/lv_3 unknown None 107G 0s ago 2022-03-08T18:42:28.428 INFO:teuthology.orchestra.run.smithi002.stdout:smithi002 /dev/vg_nvme/lv_4 unknown None 107G 0s ago 2022-03-08T18:42:28.428 INFO:teuthology.orchestra.run.smithi002.stdout:smithi066 /dev/vg_nvme/lv_1 unknown None 107G 0s ago 2022-03-08T18:42:28.428 INFO:teuthology.orchestra.run.smithi002.stdout:smithi066 /dev/vg_nvme/lv_2 unknown None 107G 0s ago 2022-03-08T18:42:28.429 INFO:teuthology.orchestra.run.smithi002.stdout:smithi066 /dev/vg_nvme/lv_3 unknown None 107G 0s ago 2022-03-08T18:42:28.429 INFO:teuthology.orchestra.run.smithi002.stdout:smithi066 /dev/vg_nvme/lv_4 unknown None 107G 0s ago 2022-03-08T18:42:28.429 INFO:teuthology.orchestra.run.smithi002.stdout:smithi137 /dev/vg_nvme/lv_1 unknown None 107G 0s ago 2022-03-08T18:42:28.429 INFO:teuthology.orchestra.run.smithi002.stdout:smithi137 /dev/vg_nvme/lv_2 unknown None 107G 0s ago 2022-03-08T18:42:28.429 INFO:teuthology.orchestra.run.smithi002.stdout:smithi137 /dev/vg_nvme/lv_3 unknown None 107G 0s ago 2022-03-08T18:42:28.430 INFO:teuthology.orchestra.run.smithi002.stdout:smithi137 /dev/vg_nvme/lv_4 unknown None 107G 0s ago 2022-03-08T18:42:28.487 INFO:teuthology.run_tasks:Running task install... 2022-03-08T18:42:28.499 DEBUG:teuthology.task.install:project ceph 2022-03-08T18:42:28.499 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '6fe8e680a334072b1ba19b2de006862f98aedbae'}} 2022-03-08T18:42:28.500 DEBUG:teuthology.task.install:config {'host.a': None, 'flavor': 'default', 'sha1': '6fe8e680a334072b1ba19b2de006862f98aedbae'} 2022-03-08T18:42:28.500 INFO:teuthology.task.install:Using flavor: default 2022-03-08T18:42:28.507 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2022-03-08T18:42:28.507 INFO:teuthology.task.install:extra packages: [] 2022-03-08T18:42:28.507 DEBUG:teuthology.orchestra.run.smithi002:> sudo lsb_release -is 2022-03-08T18:42:28.552 INFO:teuthology.orchestra.run.smithi002.stdout:Ubuntu 2022-03-08T18:42:28.557 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-03-08T18:42:28.557 DEBUG:teuthology.orchestra.run.smithi066:> sudo lsb_release -is 2022-03-08T18:42:28.558 DEBUG:teuthology.orchestra.run.smithi002:> sudo apt-key list | grep Ceph 2022-03-08T18:42:28.609 INFO:teuthology.orchestra.run.smithi066.stdout:Ubuntu 2022-03-08T18:42:28.615 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-03-08T18:42:28.616 DEBUG:teuthology.orchestra.run.smithi137:> sudo lsb_release -is 2022-03-08T18:42:28.617 DEBUG:teuthology.orchestra.run.smithi066:> sudo apt-key list | grep Ceph 2022-03-08T18:42:28.664 INFO:teuthology.orchestra.run.smithi137.stdout:Ubuntu 2022-03-08T18:42:28.669 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-03-08T18:42:28.669 DEBUG:teuthology.orchestra.run.smithi137:> sudo apt-key list | grep Ceph 2022-03-08T18:42:28.689 INFO:teuthology.orchestra.run.smithi002.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2022-03-08T18:42:28.719 INFO:teuthology.orchestra.run.smithi066.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2022-03-08T18:42:28.757 INFO:teuthology.orchestra.run.smithi137.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2022-03-08T18:42:28.776 INFO:teuthology.orchestra.run.smithi066.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2022-03-08T18:42:28.777 INFO:teuthology.orchestra.run.smithi066.stdout:uid [ unknown] Ceph.com (release key) 2022-03-08T18:42:28.778 DEBUG:teuthology.orchestra.run.smithi066:> sudo lsb_release -is 2022-03-08T18:42:28.786 INFO:teuthology.orchestra.run.smithi002.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2022-03-08T18:42:28.786 INFO:teuthology.orchestra.run.smithi002.stdout:uid [ unknown] Ceph.com (release key) 2022-03-08T18:42:28.787 DEBUG:teuthology.orchestra.run.smithi002:> sudo lsb_release -is 2022-03-08T18:42:28.817 INFO:teuthology.orchestra.run.smithi137.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2022-03-08T18:42:28.818 INFO:teuthology.orchestra.run.smithi137.stdout:uid [ unknown] Ceph.com (release key) 2022-03-08T18:42:28.819 DEBUG:teuthology.orchestra.run.smithi137:> sudo lsb_release -is 2022-03-08T18:42:28.825 INFO:teuthology.orchestra.run.smithi066.stdout:Ubuntu 2022-03-08T18:42:28.831 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-03-08T18:42:28.832 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2022-03-08T18:42:28.832 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&sha1=6fe8e680a334072b1ba19b2de006862f98aedbae 2022-03-08T18:42:28.837 INFO:teuthology.orchestra.run.smithi002.stdout:Ubuntu 2022-03-08T18:42:28.838 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-03-08T18:42:28.838 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2022-03-08T18:42:28.839 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&sha1=6fe8e680a334072b1ba19b2de006862f98aedbae 2022-03-08T18:42:28.876 INFO:teuthology.orchestra.run.smithi137.stdout:Ubuntu 2022-03-08T18:42:28.877 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-03-08T18:42:28.877 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2022-03-08T18:42:28.877 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&sha1=6fe8e680a334072b1ba19b2de006862f98aedbae 2022-03-08T18:42:28.996 INFO:teuthology.task.install.deb:Pulling from https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default/ 2022-03-08T18:42:28.996 INFO:teuthology.task.install.deb:Package version is 17.0.0-10961-g6fe8e680-1focal 2022-03-08T18:42:29.002 INFO:teuthology.task.install.deb:Pulling from https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default/ 2022-03-08T18:42:29.002 INFO:teuthology.task.install.deb:Package version is 17.0.0-10961-g6fe8e680-1focal 2022-03-08T18:42:29.038 INFO:teuthology.task.install.deb:Pulling from https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default/ 2022-03-08T18:42:29.039 INFO:teuthology.task.install.deb:Package version is 17.0.0-10961-g6fe8e680-1focal 2022-03-08T18:42:29.125 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:42:29.126 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2022-03-08T18:42:29.139 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:42:29.139 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2022-03-08T18:42:29.141 DEBUG:teuthology.orchestra.run.smithi066:> sudo apt-get update 2022-03-08T18:42:29.153 DEBUG:teuthology.orchestra.run.smithi002:> sudo apt-get update 2022-03-08T18:42:29.164 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:42:29.164 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2022-03-08T18:42:29.181 DEBUG:teuthology.orchestra.run.smithi137:> sudo apt-get update 2022-03-08T18:42:29.347 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2022-03-08T18:42:29.352 INFO:teuthology.orchestra.run.smithi066.stdout:Ign:1 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal InRelease 2022-03-08T18:42:29.375 INFO:teuthology.orchestra.run.smithi066.stdout:Get:2 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal Release [7,662 B] 2022-03-08T18:42:29.376 INFO:teuthology.orchestra.run.smithi137.stdout:Ign:1 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal InRelease 2022-03-08T18:42:29.396 INFO:teuthology.orchestra.run.smithi002.stdout:Ign:2 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal InRelease 2022-03-08T18:42:29.399 INFO:teuthology.orchestra.run.smithi137.stdout:Get:2 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal Release [7,662 B] 2022-03-08T18:42:29.400 INFO:teuthology.orchestra.run.smithi066.stdout:Ign:3 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal Release.gpg 2022-03-08T18:42:29.418 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:42:29.419 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:5 http://security.ubuntu.com/ubuntu focal-security InRelease 2022-03-08T18:42:29.419 INFO:teuthology.orchestra.run.smithi002.stdout:Get:3 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal Release [7,662 B] 2022-03-08T18:42:29.423 INFO:teuthology.orchestra.run.smithi066.stdout:Get:6 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main i386 Packages [3,342 B] 2022-03-08T18:42:29.424 INFO:teuthology.orchestra.run.smithi137.stdout:Ign:3 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal Release.gpg 2022-03-08T18:42:29.445 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:42:29.445 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:5 http://security.ubuntu.com/ubuntu focal-security InRelease 2022-03-08T18:42:29.445 INFO:teuthology.orchestra.run.smithi002.stdout:Ign:4 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal Release.gpg 2022-03-08T18:42:29.447 INFO:teuthology.orchestra.run.smithi137.stdout:Get:6 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main i386 Packages [3,342 B] 2022-03-08T18:42:29.460 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:5 http://archive.ubuntu.com/ubuntu focal InRelease 2022-03-08T18:42:29.460 INFO:teuthology.orchestra.run.smithi066.stdout:Get:7 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 Packages [17.3 kB] 2022-03-08T18:42:29.468 INFO:teuthology.orchestra.run.smithi002.stdout:Get:6 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main i386 Packages [3,342 B] 2022-03-08T18:42:29.481 INFO:teuthology.orchestra.run.smithi137.stdout:Get:7 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 Packages [17.3 kB] 2022-03-08T18:42:29.490 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:8 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2022-03-08T18:42:29.500 INFO:teuthology.orchestra.run.smithi002.stdout:Get:7 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 Packages [17.3 kB] 2022-03-08T18:42:29.516 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:8 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2022-03-08T18:42:29.543 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:8 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2022-03-08T18:42:29.573 INFO:teuthology.orchestra.run.smithi066.stdout:Hit:9 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2022-03-08T18:42:29.598 INFO:teuthology.orchestra.run.smithi137.stdout:Hit:9 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2022-03-08T18:42:29.625 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:9 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2022-03-08T18:42:29.917 INFO:teuthology.orchestra.run.smithi137.stdout:Get:10 https://packages.cloud.google.com/apt kubernetes-xenial InRelease [9,383 B] 2022-03-08T18:42:29.948 INFO:teuthology.orchestra.run.smithi002.stdout:Hit:10 https://packages.cloud.google.com/apt kubernetes-xenial InRelease 2022-03-08T18:42:29.951 INFO:teuthology.orchestra.run.smithi066.stdout:Get:10 https://packages.cloud.google.com/apt kubernetes-xenial InRelease [9,383 B] 2022-03-08T18:42:30.149 INFO:teuthology.orchestra.run.smithi002.stdout:Fetched 28.3 kB in 1s (37.3 kB/s) 2022-03-08T18:42:30.155 INFO:teuthology.orchestra.run.smithi066.stdout:Fetched 37.7 kB in 1s (46.5 kB/s) 2022-03-08T18:42:30.162 INFO:teuthology.orchestra.run.smithi137.stdout:Fetched 37.7 kB in 1s (48.3 kB/s) 2022-03-08T18:42:31.078 INFO:teuthology.orchestra.run.smithi066.stdout:Reading package lists... 2022-03-08T18:42:31.086 INFO:teuthology.orchestra.run.smithi137.stdout:Reading package lists... 2022-03-08T18:42:31.106 DEBUG:teuthology.orchestra.run.smithi066:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.0.0-10961-g6fe8e680-1focal cephadm=17.0.0-10961-g6fe8e680-1focal ceph-mds=17.0.0-10961-g6fe8e680-1focal ceph-mgr=17.0.0-10961-g6fe8e680-1focal ceph-common=17.0.0-10961-g6fe8e680-1focal ceph-fuse=17.0.0-10961-g6fe8e680-1focal ceph-test=17.0.0-10961-g6fe8e680-1focal ceph-volume=17.0.0-10961-g6fe8e680-1focal radosgw=17.0.0-10961-g6fe8e680-1focal python3-rados=17.0.0-10961-g6fe8e680-1focal python3-rgw=17.0.0-10961-g6fe8e680-1focal python3-cephfs=17.0.0-10961-g6fe8e680-1focal python3-rbd=17.0.0-10961-g6fe8e680-1focal libcephfs2=17.0.0-10961-g6fe8e680-1focal libcephfs-dev=17.0.0-10961-g6fe8e680-1focal librados2=17.0.0-10961-g6fe8e680-1focal librbd1=17.0.0-10961-g6fe8e680-1focal rbd-fuse=17.0.0-10961-g6fe8e680-1focal 2022-03-08T18:42:31.108 DEBUG:teuthology.orchestra.run.smithi137:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.0.0-10961-g6fe8e680-1focal cephadm=17.0.0-10961-g6fe8e680-1focal ceph-mds=17.0.0-10961-g6fe8e680-1focal ceph-mgr=17.0.0-10961-g6fe8e680-1focal ceph-common=17.0.0-10961-g6fe8e680-1focal ceph-fuse=17.0.0-10961-g6fe8e680-1focal ceph-test=17.0.0-10961-g6fe8e680-1focal ceph-volume=17.0.0-10961-g6fe8e680-1focal radosgw=17.0.0-10961-g6fe8e680-1focal python3-rados=17.0.0-10961-g6fe8e680-1focal python3-rgw=17.0.0-10961-g6fe8e680-1focal python3-cephfs=17.0.0-10961-g6fe8e680-1focal python3-rbd=17.0.0-10961-g6fe8e680-1focal libcephfs2=17.0.0-10961-g6fe8e680-1focal libcephfs-dev=17.0.0-10961-g6fe8e680-1focal librados2=17.0.0-10961-g6fe8e680-1focal librbd1=17.0.0-10961-g6fe8e680-1focal rbd-fuse=17.0.0-10961-g6fe8e680-1focal 2022-03-08T18:42:31.135 INFO:teuthology.orchestra.run.smithi002.stdout:Reading package lists... 2022-03-08T18:42:31.164 DEBUG:teuthology.orchestra.run.smithi002:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.0.0-10961-g6fe8e680-1focal cephadm=17.0.0-10961-g6fe8e680-1focal ceph-mds=17.0.0-10961-g6fe8e680-1focal ceph-mgr=17.0.0-10961-g6fe8e680-1focal ceph-common=17.0.0-10961-g6fe8e680-1focal ceph-fuse=17.0.0-10961-g6fe8e680-1focal ceph-test=17.0.0-10961-g6fe8e680-1focal ceph-volume=17.0.0-10961-g6fe8e680-1focal radosgw=17.0.0-10961-g6fe8e680-1focal python3-rados=17.0.0-10961-g6fe8e680-1focal python3-rgw=17.0.0-10961-g6fe8e680-1focal python3-cephfs=17.0.0-10961-g6fe8e680-1focal python3-rbd=17.0.0-10961-g6fe8e680-1focal libcephfs2=17.0.0-10961-g6fe8e680-1focal libcephfs-dev=17.0.0-10961-g6fe8e680-1focal librados2=17.0.0-10961-g6fe8e680-1focal librbd1=17.0.0-10961-g6fe8e680-1focal rbd-fuse=17.0.0-10961-g6fe8e680-1focal 2022-03-08T18:42:31.179 INFO:teuthology.orchestra.run.smithi137.stdout:Reading package lists... 2022-03-08T18:42:31.182 INFO:teuthology.orchestra.run.smithi066.stdout:Reading package lists... 2022-03-08T18:42:31.240 INFO:teuthology.orchestra.run.smithi002.stdout:Reading package lists... 2022-03-08T18:42:31.359 INFO:teuthology.orchestra.run.smithi137.stdout:Building dependency tree... 2022-03-08T18:42:31.359 INFO:teuthology.orchestra.run.smithi137.stdout:Reading state information... 2022-03-08T18:42:31.364 INFO:teuthology.orchestra.run.smithi066.stdout:Building dependency tree... 2022-03-08T18:42:31.365 INFO:teuthology.orchestra.run.smithi066.stdout:Reading state information... 2022-03-08T18:42:31.436 INFO:teuthology.orchestra.run.smithi002.stdout:Building dependency tree... 2022-03-08T18:42:31.437 INFO:teuthology.orchestra.run.smithi002.stdout:Reading state information... 2022-03-08T18:42:31.516 INFO:teuthology.orchestra.run.smithi137.stdout:The following packages were automatically installed and are no longer required: 2022-03-08T18:42:31.521 INFO:teuthology.orchestra.run.smithi137.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2022-03-08T18:42:31.522 INFO:teuthology.orchestra.run.smithi137.stdout:Use 'sudo apt autoremove' to remove them. 2022-03-08T18:42:31.522 INFO:teuthology.orchestra.run.smithi137.stdout:The following additional packages will be installed: 2022-03-08T18:42:31.523 INFO:teuthology.orchestra.run.smithi137.stdout: ceph-base ceph-mgr-dashboard ceph-mgr-diskprediction-local 2022-03-08T18:42:31.523 INFO:teuthology.orchestra.run.smithi137.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjq1 2022-03-08T18:42:31.524 INFO:teuthology.orchestra.run.smithi137.stdout: libjs-sphinxdoc libjs-underscore liblbfgsb0 liblttng-ust0 liblua5.3-0 2022-03-08T18:42:31.524 INFO:teuthology.orchestra.run.smithi137.stdout: liboath0 libonig5 librabbitmq4 libradosstriper1 librdkafka1 librgw2 2022-03-08T18:42:31.525 INFO:teuthology.orchestra.run.smithi137.stdout: libsqlite3-mod-ceph libxslt1.1 nvme-cli python-pastedeploy-tpl 2022-03-08T18:42:31.525 INFO:teuthology.orchestra.run.smithi137.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2022-03-08T18:42:31.526 INFO:teuthology.orchestra.run.smithi137.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2022-03-08T18:42:31.526 INFO:teuthology.orchestra.run.smithi137.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2022-03-08T18:42:31.527 INFO:teuthology.orchestra.run.smithi137.stdout: python3-html5lib python3-joblib python3-jwt python3-kubernetes 2022-03-08T18:42:31.527 INFO:teuthology.orchestra.run.smithi137.stdout: python3-logutils python3-lxml python3-mako python3-markupsafe 2022-03-08T18:42:31.528 INFO:teuthology.orchestra.run.smithi137.stdout: python3-oauthlib python3-openssl python3-packaging python3-paste 2022-03-08T18:42:31.528 INFO:teuthology.orchestra.run.smithi137.stdout: python3-pastedeploy python3-pastescript python3-pecan python3-pluggy 2022-03-08T18:42:31.528 INFO:teuthology.orchestra.run.smithi137.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2022-03-08T18:42:31.530 INFO:teuthology.orchestra.run.smithi137.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2022-03-08T18:42:31.530 INFO:teuthology.orchestra.run.smithi137.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2022-03-08T18:42:31.531 INFO:teuthology.orchestra.run.smithi137.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2022-03-08T18:42:31.531 INFO:teuthology.orchestra.run.smithi137.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2022-03-08T18:42:31.532 INFO:teuthology.orchestra.run.smithi137.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2022-03-08T18:42:31.533 INFO:teuthology.orchestra.run.smithi137.stdout: python3-websocket python3-webtest python3-werkzeug xmlstarlet 2022-03-08T18:42:31.533 INFO:teuthology.orchestra.run.smithi137.stdout:Suggested packages: 2022-03-08T18:42:31.534 INFO:teuthology.orchestra.run.smithi137.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2022-03-08T18:42:31.534 INFO:teuthology.orchestra.run.smithi137.stdout: python3-cryptography-vectors python3-genshi python3-crypto python3-lxml-dbg 2022-03-08T18:42:31.535 INFO:teuthology.orchestra.run.smithi137.stdout: python-lxml-doc python3-beaker python-mako-doc python-openssl-doc 2022-03-08T18:42:31.535 INFO:teuthology.orchestra.run.smithi137.stdout: python3-openssl-dbg httpd-wsgi libapache2-mod-python libapache2-mod-scgi 2022-03-08T18:42:31.535 INFO:teuthology.orchestra.run.smithi137.stdout: libjs-mochikit python-pecan-doc python-psutil-doc subversion 2022-03-08T18:42:31.536 INFO:teuthology.orchestra.run.smithi137.stdout: python-pyinotify-doc python-scipy-doc python3-dap python-sklearn-doc 2022-03-08T18:42:31.536 INFO:teuthology.orchestra.run.smithi137.stdout: ipython3 python-waitress-doc python-webob-doc python-webtest-doc 2022-03-08T18:42:31.537 INFO:teuthology.orchestra.run.smithi137.stdout: python-werkzeug-doc python3-termcolor python3-watchdog 2022-03-08T18:42:31.537 INFO:teuthology.orchestra.run.smithi137.stdout:Recommended packages: 2022-03-08T18:42:31.538 INFO:teuthology.orchestra.run.smithi137.stdout: btrfs-tools ceph-mgr-cephadm 2022-03-08T18:42:31.542 INFO:teuthology.orchestra.run.smithi066.stdout:The following packages were automatically installed and are no longer required: 2022-03-08T18:42:31.542 INFO:teuthology.orchestra.run.smithi066.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2022-03-08T18:42:31.543 INFO:teuthology.orchestra.run.smithi066.stdout:Use 'sudo apt autoremove' to remove them. 2022-03-08T18:42:31.543 INFO:teuthology.orchestra.run.smithi066.stdout:The following additional packages will be installed: 2022-03-08T18:42:31.544 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-base ceph-mgr-dashboard ceph-mgr-diskprediction-local 2022-03-08T18:42:31.544 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjq1 2022-03-08T18:42:31.545 INFO:teuthology.orchestra.run.smithi066.stdout: libjs-sphinxdoc libjs-underscore liblbfgsb0 liblttng-ust0 liblua5.3-0 2022-03-08T18:42:31.545 INFO:teuthology.orchestra.run.smithi066.stdout: liboath0 libonig5 librabbitmq4 libradosstriper1 librdkafka1 librgw2 2022-03-08T18:42:31.546 INFO:teuthology.orchestra.run.smithi066.stdout: libsqlite3-mod-ceph libxslt1.1 nvme-cli python-pastedeploy-tpl 2022-03-08T18:42:31.546 INFO:teuthology.orchestra.run.smithi066.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2022-03-08T18:42:31.546 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2022-03-08T18:42:31.547 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2022-03-08T18:42:31.547 INFO:teuthology.orchestra.run.smithi066.stdout: python3-html5lib python3-joblib python3-jwt python3-kubernetes 2022-03-08T18:42:31.548 INFO:teuthology.orchestra.run.smithi066.stdout: python3-logutils python3-lxml python3-mako python3-markupsafe 2022-03-08T18:42:31.548 INFO:teuthology.orchestra.run.smithi066.stdout: python3-oauthlib python3-openssl python3-packaging python3-paste 2022-03-08T18:42:31.549 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pastedeploy python3-pastescript python3-pecan python3-pluggy 2022-03-08T18:42:31.549 INFO:teuthology.orchestra.run.smithi066.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2022-03-08T18:42:31.550 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2022-03-08T18:42:31.550 INFO:teuthology.orchestra.run.smithi066.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2022-03-08T18:42:31.551 INFO:teuthology.orchestra.run.smithi066.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2022-03-08T18:42:31.551 INFO:teuthology.orchestra.run.smithi066.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2022-03-08T18:42:31.552 INFO:teuthology.orchestra.run.smithi066.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2022-03-08T18:42:31.552 INFO:teuthology.orchestra.run.smithi066.stdout: python3-websocket python3-webtest python3-werkzeug xmlstarlet 2022-03-08T18:42:31.553 INFO:teuthology.orchestra.run.smithi066.stdout:Suggested packages: 2022-03-08T18:42:31.553 INFO:teuthology.orchestra.run.smithi066.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2022-03-08T18:42:31.554 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cryptography-vectors python3-genshi python3-crypto python3-lxml-dbg 2022-03-08T18:42:31.554 INFO:teuthology.orchestra.run.smithi066.stdout: python-lxml-doc python3-beaker python-mako-doc python-openssl-doc 2022-03-08T18:42:31.555 INFO:teuthology.orchestra.run.smithi066.stdout: python3-openssl-dbg httpd-wsgi libapache2-mod-python libapache2-mod-scgi 2022-03-08T18:42:31.555 INFO:teuthology.orchestra.run.smithi066.stdout: libjs-mochikit python-pecan-doc python-psutil-doc subversion 2022-03-08T18:42:31.555 INFO:teuthology.orchestra.run.smithi066.stdout: python-pyinotify-doc python-scipy-doc python3-dap python-sklearn-doc 2022-03-08T18:42:31.556 INFO:teuthology.orchestra.run.smithi066.stdout: ipython3 python-waitress-doc python-webob-doc python-webtest-doc 2022-03-08T18:42:31.557 INFO:teuthology.orchestra.run.smithi066.stdout: python-werkzeug-doc python3-termcolor python3-watchdog 2022-03-08T18:42:31.558 INFO:teuthology.orchestra.run.smithi066.stdout:Recommended packages: 2022-03-08T18:42:31.558 INFO:teuthology.orchestra.run.smithi066.stdout: btrfs-tools ceph-mgr-cephadm 2022-03-08T18:42:31.610 INFO:teuthology.orchestra.run.smithi002.stdout:The following packages were automatically installed and are no longer required: 2022-03-08T18:42:31.614 INFO:teuthology.orchestra.run.smithi002.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2022-03-08T18:42:31.614 INFO:teuthology.orchestra.run.smithi002.stdout:Use 'sudo apt autoremove' to remove them. 2022-03-08T18:42:31.615 INFO:teuthology.orchestra.run.smithi002.stdout:The following additional packages will be installed: 2022-03-08T18:42:31.615 INFO:teuthology.orchestra.run.smithi002.stdout: ceph-base ceph-mgr-dashboard ceph-mgr-diskprediction-local 2022-03-08T18:42:31.615 INFO:teuthology.orchestra.run.smithi002.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjq1 2022-03-08T18:42:31.616 INFO:teuthology.orchestra.run.smithi002.stdout: libjs-sphinxdoc libjs-underscore liblbfgsb0 liblttng-ust0 liblua5.3-0 2022-03-08T18:42:31.617 INFO:teuthology.orchestra.run.smithi002.stdout: liboath0 libonig5 librabbitmq4 libradosstriper1 librdkafka1 librgw2 2022-03-08T18:42:31.617 INFO:teuthology.orchestra.run.smithi002.stdout: libsqlite3-mod-ceph libxslt1.1 nvme-cli python-pastedeploy-tpl 2022-03-08T18:42:31.618 INFO:teuthology.orchestra.run.smithi002.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2022-03-08T18:42:31.619 INFO:teuthology.orchestra.run.smithi002.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2022-03-08T18:42:31.619 INFO:teuthology.orchestra.run.smithi002.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2022-03-08T18:42:31.619 INFO:teuthology.orchestra.run.smithi002.stdout: python3-html5lib python3-joblib python3-jwt python3-kubernetes 2022-03-08T18:42:31.620 INFO:teuthology.orchestra.run.smithi002.stdout: python3-logutils python3-lxml python3-mako python3-markupsafe 2022-03-08T18:42:31.620 INFO:teuthology.orchestra.run.smithi002.stdout: python3-oauthlib python3-openssl python3-packaging python3-paste 2022-03-08T18:42:31.621 INFO:teuthology.orchestra.run.smithi002.stdout: python3-pastedeploy python3-pastescript python3-pecan python3-pluggy 2022-03-08T18:42:31.621 INFO:teuthology.orchestra.run.smithi002.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2022-03-08T18:42:31.622 INFO:teuthology.orchestra.run.smithi002.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2022-03-08T18:42:31.623 INFO:teuthology.orchestra.run.smithi002.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2022-03-08T18:42:31.623 INFO:teuthology.orchestra.run.smithi002.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2022-03-08T18:42:31.623 INFO:teuthology.orchestra.run.smithi002.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2022-03-08T18:42:31.625 INFO:teuthology.orchestra.run.smithi002.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2022-03-08T18:42:31.625 INFO:teuthology.orchestra.run.smithi002.stdout: python3-websocket python3-webtest python3-werkzeug xmlstarlet 2022-03-08T18:42:31.625 INFO:teuthology.orchestra.run.smithi002.stdout:Suggested packages: 2022-03-08T18:42:31.626 INFO:teuthology.orchestra.run.smithi002.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2022-03-08T18:42:31.626 INFO:teuthology.orchestra.run.smithi002.stdout: python3-cryptography-vectors python3-genshi python3-crypto python3-lxml-dbg 2022-03-08T18:42:31.627 INFO:teuthology.orchestra.run.smithi002.stdout: python-lxml-doc python3-beaker python-mako-doc python-openssl-doc 2022-03-08T18:42:31.627 INFO:teuthology.orchestra.run.smithi002.stdout: python3-openssl-dbg httpd-wsgi libapache2-mod-python libapache2-mod-scgi 2022-03-08T18:42:31.627 INFO:teuthology.orchestra.run.smithi002.stdout: libjs-mochikit python-pecan-doc python-psutil-doc subversion 2022-03-08T18:42:31.628 INFO:teuthology.orchestra.run.smithi002.stdout: python-pyinotify-doc python-scipy-doc python3-dap python-sklearn-doc 2022-03-08T18:42:31.628 INFO:teuthology.orchestra.run.smithi002.stdout: ipython3 python-waitress-doc python-webob-doc python-webtest-doc 2022-03-08T18:42:31.629 INFO:teuthology.orchestra.run.smithi002.stdout: python-werkzeug-doc python3-termcolor python3-watchdog 2022-03-08T18:42:31.630 INFO:teuthology.orchestra.run.smithi002.stdout:Recommended packages: 2022-03-08T18:42:31.631 INFO:teuthology.orchestra.run.smithi002.stdout: btrfs-tools ceph-mgr-cephadm 2022-03-08T18:42:31.633 INFO:teuthology.orchestra.run.smithi137.stdout:The following NEW packages will be installed: 2022-03-08T18:42:31.633 INFO:teuthology.orchestra.run.smithi137.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-dashboard 2022-03-08T18:42:31.634 INFO:teuthology.orchestra.run.smithi137.stdout: ceph-mgr-diskprediction-local ceph-mgr-k8sevents ceph-mgr-modules-core 2022-03-08T18:42:31.634 INFO:teuthology.orchestra.run.smithi137.stdout: ceph-mon ceph-osd ceph-test ceph-volume cephadm jq libcephfs-dev libcephfs2 2022-03-08T18:42:31.635 INFO:teuthology.orchestra.run.smithi137.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 liblttng-ust0 liblua5.3-0 2022-03-08T18:42:31.635 INFO:teuthology.orchestra.run.smithi137.stdout: liboath0 libonig5 librabbitmq4 libradosstriper1 librdkafka1 librgw2 2022-03-08T18:42:31.635 INFO:teuthology.orchestra.run.smithi137.stdout: libsqlite3-mod-ceph libxslt1.1 nvme-cli python-pastedeploy-tpl 2022-03-08T18:42:31.637 INFO:teuthology.orchestra.run.smithi137.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2022-03-08T18:42:31.638 INFO:teuthology.orchestra.run.smithi137.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2022-03-08T18:42:31.638 INFO:teuthology.orchestra.run.smithi137.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2022-03-08T18:42:31.639 INFO:teuthology.orchestra.run.smithi137.stdout: python3-html5lib python3-joblib python3-jwt python3-kubernetes 2022-03-08T18:42:31.639 INFO:teuthology.orchestra.run.smithi137.stdout: python3-logutils python3-lxml python3-mako python3-markupsafe 2022-03-08T18:42:31.640 INFO:teuthology.orchestra.run.smithi137.stdout: python3-oauthlib python3-openssl python3-packaging python3-paste 2022-03-08T18:42:31.640 INFO:teuthology.orchestra.run.smithi137.stdout: python3-pastedeploy python3-pastescript python3-pecan python3-pluggy 2022-03-08T18:42:31.641 INFO:teuthology.orchestra.run.smithi137.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2022-03-08T18:42:31.641 INFO:teuthology.orchestra.run.smithi137.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-rados 2022-03-08T18:42:31.641 INFO:teuthology.orchestra.run.smithi137.stdout: python3-rbd python3-repoze.lru python3-requests-oauthlib python3-rgw 2022-03-08T18:42:31.642 INFO:teuthology.orchestra.run.smithi137.stdout: python3-routes python3-rsa python3-scipy python3-simplegeneric 2022-03-08T18:42:31.642 INFO:teuthology.orchestra.run.smithi137.stdout: python3-simplejson python3-singledispatch python3-sklearn 2022-03-08T18:42:31.643 INFO:teuthology.orchestra.run.smithi137.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2022-03-08T18:42:31.643 INFO:teuthology.orchestra.run.smithi137.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2022-03-08T18:42:31.644 INFO:teuthology.orchestra.run.smithi137.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse xmlstarlet 2022-03-08T18:42:31.644 INFO:teuthology.orchestra.run.smithi137.stdout:The following packages will be upgraded: 2022-03-08T18:42:31.645 INFO:teuthology.orchestra.run.smithi137.stdout: librados2 librbd1 2022-03-08T18:42:31.649 INFO:teuthology.orchestra.run.smithi066.stdout:The following NEW packages will be installed: 2022-03-08T18:42:31.649 INFO:teuthology.orchestra.run.smithi066.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-dashboard 2022-03-08T18:42:31.650 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-diskprediction-local ceph-mgr-k8sevents ceph-mgr-modules-core 2022-03-08T18:42:31.650 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mon ceph-osd ceph-test ceph-volume cephadm jq libcephfs-dev libcephfs2 2022-03-08T18:42:31.650 INFO:teuthology.orchestra.run.smithi066.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 liblttng-ust0 liblua5.3-0 2022-03-08T18:42:31.651 INFO:teuthology.orchestra.run.smithi066.stdout: liboath0 libonig5 librabbitmq4 libradosstriper1 librdkafka1 librgw2 2022-03-08T18:42:31.651 INFO:teuthology.orchestra.run.smithi066.stdout: libsqlite3-mod-ceph libxslt1.1 nvme-cli python-pastedeploy-tpl 2022-03-08T18:42:31.651 INFO:teuthology.orchestra.run.smithi066.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2022-03-08T18:42:31.651 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2022-03-08T18:42:31.652 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2022-03-08T18:42:31.652 INFO:teuthology.orchestra.run.smithi066.stdout: python3-html5lib python3-joblib python3-jwt python3-kubernetes 2022-03-08T18:42:31.652 INFO:teuthology.orchestra.run.smithi066.stdout: python3-logutils python3-lxml python3-mako python3-markupsafe 2022-03-08T18:42:31.653 INFO:teuthology.orchestra.run.smithi066.stdout: python3-oauthlib python3-openssl python3-packaging python3-paste 2022-03-08T18:42:31.653 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pastedeploy python3-pastescript python3-pecan python3-pluggy 2022-03-08T18:42:31.653 INFO:teuthology.orchestra.run.smithi066.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2022-03-08T18:42:31.654 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-rados 2022-03-08T18:42:31.654 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rbd python3-repoze.lru python3-requests-oauthlib python3-rgw 2022-03-08T18:42:31.654 INFO:teuthology.orchestra.run.smithi066.stdout: python3-routes python3-rsa python3-scipy python3-simplegeneric 2022-03-08T18:42:31.654 INFO:teuthology.orchestra.run.smithi066.stdout: python3-simplejson python3-singledispatch python3-sklearn 2022-03-08T18:42:31.655 INFO:teuthology.orchestra.run.smithi066.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2022-03-08T18:42:31.655 INFO:teuthology.orchestra.run.smithi066.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2022-03-08T18:42:31.655 INFO:teuthology.orchestra.run.smithi066.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse xmlstarlet 2022-03-08T18:42:31.656 INFO:teuthology.orchestra.run.smithi066.stdout:The following packages will be upgraded: 2022-03-08T18:42:31.656 INFO:teuthology.orchestra.run.smithi066.stdout: librados2 librbd1 2022-03-08T18:42:31.734 INFO:teuthology.orchestra.run.smithi002.stdout:The following NEW packages will be installed: 2022-03-08T18:42:31.734 INFO:teuthology.orchestra.run.smithi002.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-dashboard 2022-03-08T18:42:31.735 INFO:teuthology.orchestra.run.smithi002.stdout: ceph-mgr-diskprediction-local ceph-mgr-k8sevents ceph-mgr-modules-core 2022-03-08T18:42:31.735 INFO:teuthology.orchestra.run.smithi002.stdout: ceph-mon ceph-osd ceph-test ceph-volume cephadm jq libcephfs-dev libcephfs2 2022-03-08T18:42:31.735 INFO:teuthology.orchestra.run.smithi002.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 liblttng-ust0 liblua5.3-0 2022-03-08T18:42:31.735 INFO:teuthology.orchestra.run.smithi002.stdout: liboath0 libonig5 librabbitmq4 libradosstriper1 librdkafka1 librgw2 2022-03-08T18:42:31.735 INFO:teuthology.orchestra.run.smithi002.stdout: libsqlite3-mod-ceph libxslt1.1 nvme-cli python-pastedeploy-tpl 2022-03-08T18:42:31.736 INFO:teuthology.orchestra.run.smithi002.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2022-03-08T18:42:31.736 INFO:teuthology.orchestra.run.smithi002.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2022-03-08T18:42:31.736 INFO:teuthology.orchestra.run.smithi002.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2022-03-08T18:42:31.736 INFO:teuthology.orchestra.run.smithi002.stdout: python3-html5lib python3-joblib python3-jwt python3-kubernetes 2022-03-08T18:42:31.737 INFO:teuthology.orchestra.run.smithi002.stdout: python3-logutils python3-lxml python3-mako python3-markupsafe 2022-03-08T18:42:31.737 INFO:teuthology.orchestra.run.smithi002.stdout: python3-oauthlib python3-openssl python3-packaging python3-paste 2022-03-08T18:42:31.737 INFO:teuthology.orchestra.run.smithi002.stdout: python3-pastedeploy python3-pastescript python3-pecan python3-pluggy 2022-03-08T18:42:31.737 INFO:teuthology.orchestra.run.smithi002.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2022-03-08T18:42:31.737 INFO:teuthology.orchestra.run.smithi002.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-rados 2022-03-08T18:42:31.738 INFO:teuthology.orchestra.run.smithi002.stdout: python3-rbd python3-repoze.lru python3-requests-oauthlib python3-rgw 2022-03-08T18:42:31.738 INFO:teuthology.orchestra.run.smithi002.stdout: python3-routes python3-rsa python3-scipy python3-simplegeneric 2022-03-08T18:42:31.738 INFO:teuthology.orchestra.run.smithi002.stdout: python3-simplejson python3-singledispatch python3-sklearn 2022-03-08T18:42:31.738 INFO:teuthology.orchestra.run.smithi002.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2022-03-08T18:42:31.739 INFO:teuthology.orchestra.run.smithi002.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2022-03-08T18:42:31.739 INFO:teuthology.orchestra.run.smithi002.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse xmlstarlet 2022-03-08T18:42:31.739 INFO:teuthology.orchestra.run.smithi002.stdout:The following packages will be upgraded: 2022-03-08T18:42:31.740 INFO:teuthology.orchestra.run.smithi002.stdout: librados2 librbd1 2022-03-08T18:42:31.886 INFO:teuthology.orchestra.run.smithi137.stdout:2 upgraded, 95 newly installed, 0 to remove and 85 not upgraded. 2022-03-08T18:42:31.886 INFO:teuthology.orchestra.run.smithi137.stdout:Need to get 181 MB of archives. 2022-03-08T18:42:31.886 INFO:teuthology.orchestra.run.smithi137.stdout:After this operation, 845 MB of additional disk space will be used. 2022-03-08T18:42:31.887 INFO:teuthology.orchestra.run.smithi137.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2022-03-08T18:42:31.891 INFO:teuthology.orchestra.run.smithi066.stdout:2 upgraded, 95 newly installed, 0 to remove and 85 not upgraded. 2022-03-08T18:42:31.891 INFO:teuthology.orchestra.run.smithi066.stdout:Need to get 181 MB of archives. 2022-03-08T18:42:31.892 INFO:teuthology.orchestra.run.smithi066.stdout:After this operation, 845 MB of additional disk space will be used. 2022-03-08T18:42:31.892 INFO:teuthology.orchestra.run.smithi066.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2022-03-08T18:42:31.925 INFO:teuthology.orchestra.run.smithi137.stdout:Get:2 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 17.0.0-10961-g6fe8e680-1focal [3,485 kB] 2022-03-08T18:42:31.930 INFO:teuthology.orchestra.run.smithi066.stdout:Get:2 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 17.0.0-10961-g6fe8e680-1focal [3,485 kB] 2022-03-08T18:42:31.985 INFO:teuthology.orchestra.run.smithi002.stdout:2 upgraded, 95 newly installed, 0 to remove and 85 not upgraded. 2022-03-08T18:42:31.986 INFO:teuthology.orchestra.run.smithi002.stdout:Need to get 181 MB of archives. 2022-03-08T18:42:31.986 INFO:teuthology.orchestra.run.smithi002.stdout:After this operation, 845 MB of additional disk space will be used. 2022-03-08T18:42:31.987 INFO:teuthology.orchestra.run.smithi002.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2022-03-08T18:42:32.022 INFO:teuthology.orchestra.run.smithi002.stdout:Get:2 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 17.0.0-10961-g6fe8e680-1focal [3,485 kB] 2022-03-08T18:42:32.314 INFO:teuthology.orchestra.run.smithi066.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2022-03-08T18:42:32.331 INFO:teuthology.orchestra.run.smithi066.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2022-03-08T18:42:32.332 INFO:teuthology.orchestra.run.smithi137.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2022-03-08T18:42:32.340 INFO:teuthology.orchestra.run.smithi137.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2022-03-08T18:42:32.352 INFO:teuthology.orchestra.run.smithi137.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2022-03-08T18:42:32.361 INFO:teuthology.orchestra.run.smithi066.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2022-03-08T18:42:32.402 INFO:teuthology.orchestra.run.smithi066.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2022-03-08T18:42:32.423 INFO:teuthology.orchestra.run.smithi137.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2022-03-08T18:42:32.424 INFO:teuthology.orchestra.run.smithi002.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2022-03-08T18:42:32.438 INFO:teuthology.orchestra.run.smithi002.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2022-03-08T18:42:32.473 INFO:teuthology.orchestra.run.smithi002.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2022-03-08T18:42:32.514 INFO:teuthology.orchestra.run.smithi066.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2022-03-08T18:42:32.522 INFO:teuthology.orchestra.run.smithi066.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2022-03-08T18:42:32.523 INFO:teuthology.orchestra.run.smithi002.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2022-03-08T18:42:32.539 INFO:teuthology.orchestra.run.smithi137.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2022-03-08T18:42:32.545 INFO:teuthology.orchestra.run.smithi137.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2022-03-08T18:42:32.560 INFO:teuthology.orchestra.run.smithi066.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2022-03-08T18:42:32.565 INFO:teuthology.orchestra.run.smithi066.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2022-03-08T18:42:32.567 INFO:teuthology.orchestra.run.smithi066.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2022-03-08T18:42:32.585 INFO:teuthology.orchestra.run.smithi137.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2022-03-08T18:42:32.586 INFO:teuthology.orchestra.run.smithi066.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jwt all 1.7.1-2ubuntu2 [17.4 kB] 2022-03-08T18:42:32.586 INFO:teuthology.orchestra.run.smithi066.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2022-03-08T18:42:32.590 INFO:teuthology.orchestra.run.smithi137.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2022-03-08T18:42:32.590 INFO:teuthology.orchestra.run.smithi066.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2022-03-08T18:42:32.591 INFO:teuthology.orchestra.run.smithi066.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2 [59.1 kB] 2022-03-08T18:42:32.594 INFO:teuthology.orchestra.run.smithi137.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2022-03-08T18:42:32.609 INFO:teuthology.orchestra.run.smithi137.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jwt all 1.7.1-2ubuntu2 [17.4 kB] 2022-03-08T18:42:32.616 INFO:teuthology.orchestra.run.smithi137.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2022-03-08T18:42:32.617 INFO:teuthology.orchestra.run.smithi137.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2022-03-08T18:42:32.618 INFO:teuthology.orchestra.run.smithi137.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2 [59.1 kB] 2022-03-08T18:42:32.625 INFO:teuthology.orchestra.run.smithi137.stdout:Get:16 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 17.0.0-10961-g6fe8e680-1focal [3,532 kB] 2022-03-08T18:42:32.637 INFO:teuthology.orchestra.run.smithi002.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2022-03-08T18:42:32.646 INFO:teuthology.orchestra.run.smithi002.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2022-03-08T18:42:32.647 INFO:teuthology.orchestra.run.smithi066.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2022-03-08T18:42:32.647 INFO:teuthology.orchestra.run.smithi066.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2022-03-08T18:42:32.648 INFO:teuthology.orchestra.run.smithi066.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2022-03-08T18:42:32.668 INFO:teuthology.orchestra.run.smithi137.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2022-03-08T18:42:32.668 INFO:teuthology.orchestra.run.smithi137.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2022-03-08T18:42:32.677 INFO:teuthology.orchestra.run.smithi002.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2022-03-08T18:42:32.684 INFO:teuthology.orchestra.run.smithi002.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2022-03-08T18:42:32.688 INFO:teuthology.orchestra.run.smithi002.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2022-03-08T18:42:32.707 INFO:teuthology.orchestra.run.smithi002.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jwt all 1.7.1-2ubuntu2 [17.4 kB] 2022-03-08T18:42:32.708 INFO:teuthology.orchestra.run.smithi002.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2022-03-08T18:42:32.709 INFO:teuthology.orchestra.run.smithi002.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2022-03-08T18:42:32.713 INFO:teuthology.orchestra.run.smithi002.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2 [59.1 kB] 2022-03-08T18:42:32.725 INFO:teuthology.orchestra.run.smithi066.stdout:Get:19 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 17.0.0-10961-g6fe8e680-1focal [3,532 kB] 2022-03-08T18:42:32.726 INFO:teuthology.orchestra.run.smithi002.stdout:Get:16 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 17.0.0-10961-g6fe8e680-1focal [3,532 kB] 2022-03-08T18:42:32.730 INFO:teuthology.orchestra.run.smithi066.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2022-03-08T18:42:32.731 INFO:teuthology.orchestra.run.smithi066.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2022-03-08T18:42:32.743 INFO:teuthology.orchestra.run.smithi066.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-waitress all 1.4.1-1 [80.6 kB] 2022-03-08T18:42:32.744 INFO:teuthology.orchestra.run.smithi066.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2022-03-08T18:42:32.744 INFO:teuthology.orchestra.run.smithi066.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2022-03-08T18:42:32.756 INFO:teuthology.orchestra.run.smithi137.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2022-03-08T18:42:32.758 INFO:teuthology.orchestra.run.smithi066.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2022-03-08T18:42:32.758 INFO:teuthology.orchestra.run.smithi066.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2022-03-08T18:42:32.763 INFO:teuthology.orchestra.run.smithi137.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2022-03-08T18:42:32.763 INFO:teuthology.orchestra.run.smithi137.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2022-03-08T18:42:32.765 INFO:teuthology.orchestra.run.smithi137.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-waitress all 1.4.1-1 [80.6 kB] 2022-03-08T18:42:32.767 INFO:teuthology.orchestra.run.smithi137.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2022-03-08T18:42:32.768 INFO:teuthology.orchestra.run.smithi137.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2022-03-08T18:42:32.768 INFO:teuthology.orchestra.run.smithi002.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2022-03-08T18:42:32.769 INFO:teuthology.orchestra.run.smithi002.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2022-03-08T18:42:32.780 INFO:teuthology.orchestra.run.smithi137.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2022-03-08T18:42:32.780 INFO:teuthology.orchestra.run.smithi137.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2022-03-08T18:42:32.819 INFO:teuthology.orchestra.run.smithi066.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2022-03-08T18:42:32.820 INFO:teuthology.orchestra.run.smithi066.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2022-03-08T18:42:32.824 INFO:teuthology.orchestra.run.smithi066.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-werkzeug all 0.16.1+dfsg1-2 [183 kB] 2022-03-08T18:42:32.842 INFO:teuthology.orchestra.run.smithi137.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2022-03-08T18:42:32.846 INFO:teuthology.orchestra.run.smithi137.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2022-03-08T18:42:32.853 INFO:teuthology.orchestra.run.smithi002.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2022-03-08T18:42:32.905 INFO:teuthology.orchestra.run.smithi066.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2022-03-08T18:42:32.906 INFO:teuthology.orchestra.run.smithi066.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2022-03-08T18:42:32.910 INFO:teuthology.orchestra.run.smithi066.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2022-03-08T18:42:32.914 INFO:teuthology.orchestra.run.smithi002.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2022-03-08T18:42:32.915 INFO:teuthology.orchestra.run.smithi066.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2022-03-08T18:42:32.915 INFO:teuthology.orchestra.run.smithi002.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2022-03-08T18:42:32.917 INFO:teuthology.orchestra.run.smithi066.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2022-03-08T18:42:32.919 INFO:teuthology.orchestra.run.smithi066.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2022-03-08T18:42:32.919 INFO:teuthology.orchestra.run.smithi066.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2022-03-08T18:42:32.926 INFO:teuthology.orchestra.run.smithi137.stdout:Get:29 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 17.0.0-10961-g6fe8e680-1focal [752 kB] 2022-03-08T18:42:32.935 INFO:teuthology.orchestra.run.smithi137.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-werkzeug all 0.16.1+dfsg1-2 [183 kB] 2022-03-08T18:42:32.942 INFO:teuthology.orchestra.run.smithi137.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2022-03-08T18:42:32.943 INFO:teuthology.orchestra.run.smithi137.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2022-03-08T18:42:32.947 INFO:teuthology.orchestra.run.smithi137.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2022-03-08T18:42:32.949 INFO:teuthology.orchestra.run.smithi002.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-waitress all 1.4.1-1 [80.6 kB] 2022-03-08T18:42:32.951 INFO:teuthology.orchestra.run.smithi002.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2022-03-08T18:42:32.952 INFO:teuthology.orchestra.run.smithi002.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2022-03-08T18:42:32.954 INFO:teuthology.orchestra.run.smithi137.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2022-03-08T18:42:32.956 INFO:teuthology.orchestra.run.smithi002.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2022-03-08T18:42:32.957 INFO:teuthology.orchestra.run.smithi137.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2022-03-08T18:42:32.957 INFO:teuthology.orchestra.run.smithi137.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2022-03-08T18:42:32.960 INFO:teuthology.orchestra.run.smithi137.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2022-03-08T18:42:32.960 INFO:teuthology.orchestra.run.smithi002.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2022-03-08T18:42:32.962 INFO:teuthology.orchestra.run.smithi002.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2022-03-08T18:42:32.969 INFO:teuthology.orchestra.run.smithi002.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2022-03-08T18:42:32.995 INFO:teuthology.orchestra.run.smithi137.stdout:Get:38 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 17.0.0-10961-g6fe8e680-1focal [412 kB] 2022-03-08T18:42:33.033 INFO:teuthology.orchestra.run.smithi137.stdout:Get:39 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 17.0.0-10961-g6fe8e680-1focal [22.4 kB] 2022-03-08T18:42:33.034 INFO:teuthology.orchestra.run.smithi137.stdout:Get:40 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 17.0.0-10961-g6fe8e680-1focal [200 kB] 2022-03-08T18:42:33.063 INFO:teuthology.orchestra.run.smithi002.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-werkzeug all 0.16.1+dfsg1-2 [183 kB] 2022-03-08T18:42:33.072 INFO:teuthology.orchestra.run.smithi002.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2022-03-08T18:42:33.073 INFO:teuthology.orchestra.run.smithi002.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2022-03-08T18:42:33.077 INFO:teuthology.orchestra.run.smithi137.stdout:Get:41 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 17.0.0-10961-g6fe8e680-1focal [41.9 kB] 2022-03-08T18:42:33.079 INFO:teuthology.orchestra.run.smithi002.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2022-03-08T18:42:33.079 INFO:teuthology.orchestra.run.smithi137.stdout:Get:42 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 17.0.0-10961-g6fe8e680-1focal [385 kB] 2022-03-08T18:42:33.085 INFO:teuthology.orchestra.run.smithi002.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2022-03-08T18:42:33.134 INFO:teuthology.orchestra.run.smithi137.stdout:Get:43 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 17.0.0-10961-g6fe8e680-1focal [4,161 kB] 2022-03-08T18:42:33.158 INFO:teuthology.orchestra.run.smithi066.stdout:Get:37 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 17.0.0-10961-g6fe8e680-1focal [752 kB] 2022-03-08T18:42:33.172 INFO:teuthology.orchestra.run.smithi002.stdout:Get:34 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 17.0.0-10961-g6fe8e680-1focal [752 kB] 2022-03-08T18:42:33.176 INFO:teuthology.orchestra.run.smithi002.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2022-03-08T18:42:33.177 INFO:teuthology.orchestra.run.smithi002.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2022-03-08T18:42:33.177 INFO:teuthology.orchestra.run.smithi002.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2022-03-08T18:42:33.332 INFO:teuthology.orchestra.run.smithi002.stdout:Get:38 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 17.0.0-10961-g6fe8e680-1focal [412 kB] 2022-03-08T18:42:33.340 INFO:teuthology.orchestra.run.smithi066.stdout:Get:38 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 17.0.0-10961-g6fe8e680-1focal [412 kB] 2022-03-08T18:42:33.346 INFO:teuthology.orchestra.run.smithi066.stdout:Get:39 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 17.0.0-10961-g6fe8e680-1focal [22.4 kB] 2022-03-08T18:42:33.353 INFO:teuthology.orchestra.run.smithi066.stdout:Get:40 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 17.0.0-10961-g6fe8e680-1focal [200 kB] 2022-03-08T18:42:33.356 INFO:teuthology.orchestra.run.smithi002.stdout:Get:39 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 17.0.0-10961-g6fe8e680-1focal [22.4 kB] 2022-03-08T18:42:33.357 INFO:teuthology.orchestra.run.smithi002.stdout:Get:40 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 17.0.0-10961-g6fe8e680-1focal [200 kB] 2022-03-08T18:42:33.363 INFO:teuthology.orchestra.run.smithi002.stdout:Get:41 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 17.0.0-10961-g6fe8e680-1focal [41.9 kB] 2022-03-08T18:42:33.381 INFO:teuthology.orchestra.run.smithi002.stdout:Get:42 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 17.0.0-10961-g6fe8e680-1focal [385 kB] 2022-03-08T18:42:33.386 INFO:teuthology.orchestra.run.smithi066.stdout:Get:41 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 17.0.0-10961-g6fe8e680-1focal [41.9 kB] 2022-03-08T18:42:33.386 INFO:teuthology.orchestra.run.smithi066.stdout:Get:42 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 17.0.0-10961-g6fe8e680-1focal [385 kB] 2022-03-08T18:42:33.415 INFO:teuthology.orchestra.run.smithi066.stdout:Get:43 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 17.0.0-10961-g6fe8e680-1focal [4,161 kB] 2022-03-08T18:42:33.418 INFO:teuthology.orchestra.run.smithi002.stdout:Get:43 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 17.0.0-10961-g6fe8e680-1focal [4,161 kB] 2022-03-08T18:42:33.493 INFO:teuthology.orchestra.run.smithi137.stdout:Get:44 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 17.0.0-10961-g6fe8e680-1focal [116 kB] 2022-03-08T18:42:33.498 INFO:teuthology.orchestra.run.smithi066.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2022-03-08T18:42:33.503 INFO:teuthology.orchestra.run.smithi137.stdout:Get:45 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 17.0.0-10961-g6fe8e680-1focal [478 kB] 2022-03-08T18:42:33.529 INFO:teuthology.orchestra.run.smithi137.stdout:Get:46 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 17.0.0-10961-g6fe8e680-1focal [24.4 MB] 2022-03-08T18:42:33.562 INFO:teuthology.orchestra.run.smithi137.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2022-03-08T18:42:33.685 INFO:teuthology.orchestra.run.smithi066.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2022-03-08T18:42:33.712 INFO:teuthology.orchestra.run.smithi066.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2022-03-08T18:42:33.720 INFO:teuthology.orchestra.run.smithi137.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2022-03-08T18:42:33.722 INFO:teuthology.orchestra.run.smithi137.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2022-03-08T18:42:33.731 INFO:teuthology.orchestra.run.smithi066.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2022-03-08T18:42:33.731 INFO:teuthology.orchestra.run.smithi066.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2022-03-08T18:42:33.731 INFO:teuthology.orchestra.run.smithi066.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2022-03-08T18:42:33.732 INFO:teuthology.orchestra.run.smithi066.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2022-03-08T18:42:33.738 INFO:teuthology.orchestra.run.smithi066.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2022-03-08T18:42:33.738 INFO:teuthology.orchestra.run.smithi066.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2022-03-08T18:42:33.738 INFO:teuthology.orchestra.run.smithi066.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2022-03-08T18:42:33.768 INFO:teuthology.orchestra.run.smithi066.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2022-03-08T18:42:33.804 INFO:teuthology.orchestra.run.smithi002.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2022-03-08T18:42:33.817 INFO:teuthology.orchestra.run.smithi137.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2022-03-08T18:42:33.818 INFO:teuthology.orchestra.run.smithi137.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2022-03-08T18:42:33.819 INFO:teuthology.orchestra.run.smithi137.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2022-03-08T18:42:33.819 INFO:teuthology.orchestra.run.smithi137.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2022-03-08T18:42:33.820 INFO:teuthology.orchestra.run.smithi137.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2022-03-08T18:42:33.821 INFO:teuthology.orchestra.run.smithi137.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2022-03-08T18:42:33.822 INFO:teuthology.orchestra.run.smithi137.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2022-03-08T18:42:33.822 INFO:teuthology.orchestra.run.smithi137.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2022-03-08T18:42:33.833 INFO:teuthology.orchestra.run.smithi066.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2022-03-08T18:42:33.834 INFO:teuthology.orchestra.run.smithi066.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2022-03-08T18:42:33.842 INFO:teuthology.orchestra.run.smithi066.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2022-03-08T18:42:33.845 INFO:teuthology.orchestra.run.smithi066.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2022-03-08T18:42:33.848 INFO:teuthology.orchestra.run.smithi066.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2022-03-08T18:42:33.848 INFO:teuthology.orchestra.run.smithi066.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 libxslt1.1 amd64 1.1.34-4 [152 kB] 2022-03-08T18:42:33.851 INFO:teuthology.orchestra.run.smithi066.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2022-03-08T18:42:33.855 INFO:teuthology.orchestra.run.smithi066.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2022-03-08T18:42:33.866 INFO:teuthology.orchestra.run.smithi066.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2022-03-08T18:42:33.888 INFO:teuthology.orchestra.run.smithi002.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2022-03-08T18:42:33.892 INFO:teuthology.orchestra.run.smithi002.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2022-03-08T18:42:33.916 INFO:teuthology.orchestra.run.smithi137.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2022-03-08T18:42:33.920 INFO:teuthology.orchestra.run.smithi137.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2022-03-08T18:42:33.928 INFO:teuthology.orchestra.run.smithi137.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2022-03-08T18:42:33.932 INFO:teuthology.orchestra.run.smithi137.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2022-03-08T18:42:33.936 INFO:teuthology.orchestra.run.smithi137.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2022-03-08T18:42:33.936 INFO:teuthology.orchestra.run.smithi137.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/main amd64 libxslt1.1 amd64 1.1.34-4 [152 kB] 2022-03-08T18:42:33.938 INFO:teuthology.orchestra.run.smithi002.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2022-03-08T18:42:33.939 INFO:teuthology.orchestra.run.smithi002.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2022-03-08T18:42:33.940 INFO:teuthology.orchestra.run.smithi002.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2022-03-08T18:42:33.941 INFO:teuthology.orchestra.run.smithi002.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2022-03-08T18:42:33.941 INFO:teuthology.orchestra.run.smithi002.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2022-03-08T18:42:33.942 INFO:teuthology.orchestra.run.smithi137.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2022-03-08T18:42:33.942 INFO:teuthology.orchestra.run.smithi002.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2022-03-08T18:42:33.943 INFO:teuthology.orchestra.run.smithi002.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2022-03-08T18:42:33.946 INFO:teuthology.orchestra.run.smithi137.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2022-03-08T18:42:33.952 INFO:teuthology.orchestra.run.smithi137.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2022-03-08T18:42:33.959 INFO:teuthology.orchestra.run.smithi066.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2022-03-08T18:42:33.960 INFO:teuthology.orchestra.run.smithi066.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2022-03-08T18:42:33.961 INFO:teuthology.orchestra.run.smithi066.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2022-03-08T18:42:33.962 INFO:teuthology.orchestra.run.smithi066.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2022-03-08T18:42:33.979 INFO:teuthology.orchestra.run.smithi066.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2022-03-08T18:42:33.980 INFO:teuthology.orchestra.run.smithi066.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2022-03-08T18:42:33.981 INFO:teuthology.orchestra.run.smithi066.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2022-03-08T18:42:33.981 INFO:teuthology.orchestra.run.smithi066.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2022-03-08T18:42:33.983 INFO:teuthology.orchestra.run.smithi066.stdout:Get:72 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2022-03-08T18:42:34.002 INFO:teuthology.orchestra.run.smithi002.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2022-03-08T18:42:34.003 INFO:teuthology.orchestra.run.smithi002.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2022-03-08T18:42:34.006 INFO:teuthology.orchestra.run.smithi137.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2022-03-08T18:42:34.035 INFO:teuthology.orchestra.run.smithi137.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2022-03-08T18:42:34.035 INFO:teuthology.orchestra.run.smithi137.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2022-03-08T18:42:34.038 INFO:teuthology.orchestra.run.smithi137.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2022-03-08T18:42:34.055 INFO:teuthology.orchestra.run.smithi066.stdout:Get:73 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2022-03-08T18:42:34.056 INFO:teuthology.orchestra.run.smithi066.stdout:Get:74 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2022-03-08T18:42:34.057 INFO:teuthology.orchestra.run.smithi066.stdout:Get:75 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2022-03-08T18:42:34.059 INFO:teuthology.orchestra.run.smithi066.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2022-03-08T18:42:34.078 INFO:teuthology.orchestra.run.smithi137.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2022-03-08T18:42:34.078 INFO:teuthology.orchestra.run.smithi137.stdout:Get:72 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2022-03-08T18:42:34.080 INFO:teuthology.orchestra.run.smithi137.stdout:Get:73 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2022-03-08T18:42:34.080 INFO:teuthology.orchestra.run.smithi137.stdout:Get:74 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2022-03-08T18:42:34.083 INFO:teuthology.orchestra.run.smithi002.stdout:Get:56 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 17.0.0-10961-g6fe8e680-1focal [116 kB] 2022-03-08T18:42:34.084 INFO:teuthology.orchestra.run.smithi002.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2022-03-08T18:42:34.085 INFO:teuthology.orchestra.run.smithi137.stdout:Get:75 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2022-03-08T18:42:34.089 INFO:teuthology.orchestra.run.smithi002.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2022-03-08T18:42:34.090 INFO:teuthology.orchestra.run.smithi137.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2022-03-08T18:42:34.092 INFO:teuthology.orchestra.run.smithi002.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2022-03-08T18:42:34.101 INFO:teuthology.orchestra.run.smithi002.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2022-03-08T18:42:34.102 INFO:teuthology.orchestra.run.smithi002.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/main amd64 libxslt1.1 amd64 1.1.34-4 [152 kB] 2022-03-08T18:42:34.102 INFO:teuthology.orchestra.run.smithi002.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2022-03-08T18:42:34.102 INFO:teuthology.orchestra.run.smithi002.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2022-03-08T18:42:34.103 INFO:teuthology.orchestra.run.smithi002.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2022-03-08T18:42:34.111 INFO:teuthology.orchestra.run.smithi066.stdout:Get:77 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 17.0.0-10961-g6fe8e680-1focal [116 kB] 2022-03-08T18:42:34.114 INFO:teuthology.orchestra.run.smithi002.stdout:Get:65 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 17.0.0-10961-g6fe8e680-1focal [478 kB] 2022-03-08T18:42:34.114 INFO:teuthology.orchestra.run.smithi066.stdout:Get:78 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 17.0.0-10961-g6fe8e680-1focal [478 kB] 2022-03-08T18:42:34.166 INFO:teuthology.orchestra.run.smithi002.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2022-03-08T18:42:34.167 INFO:teuthology.orchestra.run.smithi002.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2022-03-08T18:42:34.173 INFO:teuthology.orchestra.run.smithi137.stdout:Get:77 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2022-03-08T18:42:34.174 INFO:teuthology.orchestra.run.smithi137.stdout:Get:78 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2022-03-08T18:42:34.176 INFO:teuthology.orchestra.run.smithi066.stdout:Get:79 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 17.0.0-10961-g6fe8e680-1focal [24.4 MB] 2022-03-08T18:42:34.177 INFO:teuthology.orchestra.run.smithi002.stdout:Get:68 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 17.0.0-10961-g6fe8e680-1focal [24.4 MB] 2022-03-08T18:42:34.180 INFO:teuthology.orchestra.run.smithi137.stdout:Get:79 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2022-03-08T18:42:34.249 INFO:teuthology.orchestra.run.smithi002.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2022-03-08T18:42:34.250 INFO:teuthology.orchestra.run.smithi002.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2022-03-08T18:42:34.274 INFO:teuthology.orchestra.run.smithi002.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2022-03-08T18:42:34.274 INFO:teuthology.orchestra.run.smithi002.stdout:Get:72 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2022-03-08T18:42:34.275 INFO:teuthology.orchestra.run.smithi002.stdout:Get:73 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2022-03-08T18:42:34.275 INFO:teuthology.orchestra.run.smithi002.stdout:Get:74 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2022-03-08T18:42:34.278 INFO:teuthology.orchestra.run.smithi002.stdout:Get:75 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2022-03-08T18:42:34.279 INFO:teuthology.orchestra.run.smithi002.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2022-03-08T18:42:34.331 INFO:teuthology.orchestra.run.smithi002.stdout:Get:77 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2022-03-08T18:42:34.332 INFO:teuthology.orchestra.run.smithi002.stdout:Get:78 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2022-03-08T18:42:34.413 INFO:teuthology.orchestra.run.smithi002.stdout:Get:79 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2022-03-08T18:42:35.167 INFO:teuthology.orchestra.run.smithi137.stdout:Get:80 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 17.0.0-10961-g6fe8e680-1focal [6,428 kB] 2022-03-08T18:42:35.498 INFO:teuthology.orchestra.run.smithi137.stdout:Get:81 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 17.0.0-10961-g6fe8e680-1focal [210 kB] 2022-03-08T18:42:35.503 INFO:teuthology.orchestra.run.smithi137.stdout:Get:82 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 17.0.0-10961-g6fe8e680-1focal [151 kB] 2022-03-08T18:42:35.508 INFO:teuthology.orchestra.run.smithi137.stdout:Get:83 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 17.0.0-10961-g6fe8e680-1focal [1,570 kB] 2022-03-08T18:42:35.613 INFO:teuthology.orchestra.run.smithi137.stdout:Get:84 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 17.0.0-10961-g6fe8e680-1focal [7,816 kB] 2022-03-08T18:42:36.082 INFO:teuthology.orchestra.run.smithi137.stdout:Get:85 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 17.0.0-10961-g6fe8e680-1focal [28.6 MB] 2022-03-08T18:42:37.816 INFO:teuthology.orchestra.run.smithi002.stdout:Get:80 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 17.0.0-10961-g6fe8e680-1focal [6,428 kB] 2022-03-08T18:42:37.817 INFO:teuthology.orchestra.run.smithi066.stdout:Get:80 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 17.0.0-10961-g6fe8e680-1focal [6,428 kB] 2022-03-08T18:42:37.955 INFO:teuthology.orchestra.run.smithi137.stdout:Get:86 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 17.0.0-10961-g6fe8e680-1focal [3,812 B] 2022-03-08T18:42:37.955 INFO:teuthology.orchestra.run.smithi137.stdout:Get:87 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 17.0.0-10961-g6fe8e680-1focal [859 kB] 2022-03-08T18:42:37.960 INFO:teuthology.orchestra.run.smithi137.stdout:Get:88 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 17.0.0-10961-g6fe8e680-1focal [2,385 kB] 2022-03-08T18:42:38.048 INFO:teuthology.orchestra.run.smithi137.stdout:Get:89 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 17.0.0-10961-g6fe8e680-1focal [1,677 kB] 2022-03-08T18:42:38.177 INFO:teuthology.orchestra.run.smithi137.stdout:Get:90 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 17.0.0-10961-g6fe8e680-1focal [7,876 kB] 2022-03-08T18:42:38.681 INFO:teuthology.orchestra.run.smithi137.stdout:Get:91 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 17.0.0-10961-g6fe8e680-1focal [3,932 B] 2022-03-08T18:42:38.694 INFO:teuthology.orchestra.run.smithi002.stdout:Get:81 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 17.0.0-10961-g6fe8e680-1focal [210 kB] 2022-03-08T18:42:38.719 INFO:teuthology.orchestra.run.smithi137.stdout:Get:92 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 17.0.0-10961-g6fe8e680-1focal [50.5 MB] 2022-03-08T18:42:38.731 INFO:teuthology.orchestra.run.smithi002.stdout:Get:82 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 17.0.0-10961-g6fe8e680-1focal [151 kB] 2022-03-08T18:42:38.736 INFO:teuthology.orchestra.run.smithi066.stdout:Get:81 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 17.0.0-10961-g6fe8e680-1focal [210 kB] 2022-03-08T18:42:38.745 INFO:teuthology.orchestra.run.smithi066.stdout:Get:82 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 17.0.0-10961-g6fe8e680-1focal [151 kB] 2022-03-08T18:42:38.757 INFO:teuthology.orchestra.run.smithi002.stdout:Get:83 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 17.0.0-10961-g6fe8e680-1focal [1,570 kB] 2022-03-08T18:42:38.783 INFO:teuthology.orchestra.run.smithi066.stdout:Get:83 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 17.0.0-10961-g6fe8e680-1focal [1,570 kB] 2022-03-08T18:42:38.975 INFO:teuthology.orchestra.run.smithi002.stdout:Get:84 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 17.0.0-10961-g6fe8e680-1focal [7,816 kB] 2022-03-08T18:42:39.019 INFO:teuthology.orchestra.run.smithi066.stdout:Get:84 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 17.0.0-10961-g6fe8e680-1focal [7,816 kB] 2022-03-08T18:42:40.153 INFO:teuthology.orchestra.run.smithi002.stdout:Get:85 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 17.0.0-10961-g6fe8e680-1focal [28.6 MB] 2022-03-08T18:42:40.165 INFO:teuthology.orchestra.run.smithi066.stdout:Get:85 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 17.0.0-10961-g6fe8e680-1focal [28.6 MB] 2022-03-08T18:42:42.128 INFO:teuthology.orchestra.run.smithi137.stdout:Get:93 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-volume all 17.0.0-10961-g6fe8e680-1focal [219 kB] 2022-03-08T18:42:42.130 INFO:teuthology.orchestra.run.smithi137.stdout:Get:94 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 17.0.0-10961-g6fe8e680-1focal [69.5 kB] 2022-03-08T18:42:42.200 INFO:teuthology.orchestra.run.smithi137.stdout:Get:95 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 17.0.0-10961-g6fe8e680-1focal [21.4 kB] 2022-03-08T18:42:42.200 INFO:teuthology.orchestra.run.smithi137.stdout:Get:96 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 17.0.0-10961-g6fe8e680-1focal [12.7 MB] 2022-03-08T18:42:42.938 INFO:teuthology.orchestra.run.smithi137.stdout:Get:97 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 17.0.0-10961-g6fe8e680-1focal [78.8 kB] 2022-03-08T18:42:43.519 INFO:teuthology.orchestra.run.smithi066.stdout:Get:86 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 17.0.0-10961-g6fe8e680-1focal [3,812 B] 2022-03-08T18:42:43.520 INFO:teuthology.orchestra.run.smithi066.stdout:Get:87 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 17.0.0-10961-g6fe8e680-1focal [859 kB] 2022-03-08T18:42:43.523 INFO:teuthology.orchestra.run.smithi002.stdout:Get:86 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 17.0.0-10961-g6fe8e680-1focal [3,812 B] 2022-03-08T18:42:43.523 INFO:teuthology.orchestra.run.smithi002.stdout:Get:87 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 17.0.0-10961-g6fe8e680-1focal [859 kB] 2022-03-08T18:42:43.530 INFO:teuthology.orchestra.run.smithi066.stdout:Get:88 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 17.0.0-10961-g6fe8e680-1focal [2,385 kB] 2022-03-08T18:42:43.531 INFO:teuthology.orchestra.run.smithi002.stdout:Get:88 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 17.0.0-10961-g6fe8e680-1focal [2,385 kB] 2022-03-08T18:42:43.565 INFO:teuthology.orchestra.run.smithi137.stdout:Fetched 181 MB in 11s (16.0 MB/s) 2022-03-08T18:42:43.734 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2022-03-08T18:42:43.780 INFO:teuthology.orchestra.run.smithi137.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 109337 files and directories currently installed.) 2022-03-08T18:42:43.783 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../00-liblttng-ust0_2.11.0-1_amd64.deb ... 2022-03-08T18:42:43.834 INFO:teuthology.orchestra.run.smithi066.stdout:Get:89 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 17.0.0-10961-g6fe8e680-1focal [1,677 kB] 2022-03-08T18:42:43.891 INFO:teuthology.orchestra.run.smithi002.stdout:Get:89 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 17.0.0-10961-g6fe8e680-1focal [1,677 kB] 2022-03-08T18:42:43.916 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2022-03-08T18:42:44.016 INFO:teuthology.orchestra.run.smithi066.stdout:Get:90 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 17.0.0-10961-g6fe8e680-1focal [7,876 kB] 2022-03-08T18:42:44.078 INFO:teuthology.orchestra.run.smithi002.stdout:Get:90 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 17.0.0-10961-g6fe8e680-1focal [7,876 kB] 2022-03-08T18:42:44.319 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../01-librbd1_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:44.447 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking librbd1 (17.0.0-10961-g6fe8e680-1focal) over (15.2.14-0ubuntu0.20.04.2) ... 2022-03-08T18:42:44.722 INFO:teuthology.orchestra.run.smithi066.stdout:Get:91 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 17.0.0-10961-g6fe8e680-1focal [3,932 B] 2022-03-08T18:42:44.733 INFO:teuthology.orchestra.run.smithi066.stdout:Get:92 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 17.0.0-10961-g6fe8e680-1focal [50.5 MB] 2022-03-08T18:42:44.774 INFO:teuthology.orchestra.run.smithi002.stdout:Get:91 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 17.0.0-10961-g6fe8e680-1focal [3,932 B] 2022-03-08T18:42:44.774 INFO:teuthology.orchestra.run.smithi002.stdout:Get:92 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 17.0.0-10961-g6fe8e680-1focal [50.5 MB] 2022-03-08T18:42:45.197 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../02-librados2_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:45.394 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking librados2 (17.0.0-10961-g6fe8e680-1focal) over (15.2.14-0ubuntu0.20.04.2) ... 2022-03-08T18:42:46.074 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libcephfs2. 2022-03-08T18:42:46.086 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../03-libcephfs2_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:46.124 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libcephfs2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:46.536 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-rados. 2022-03-08T18:42:46.550 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../04-python3-rados_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:46.602 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-rados (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:46.939 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-ceph-argparse. 2022-03-08T18:42:46.952 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../05-python3-ceph-argparse_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:42:46.980 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-ceph-argparse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:47.208 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-cephfs. 2022-03-08T18:42:47.222 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../06-python3-cephfs_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:47.266 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-cephfs (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:47.704 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-ceph-common. 2022-03-08T18:42:47.718 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../07-python3-ceph-common_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:42:47.762 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-ceph-common (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:48.099 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-prettytable. 2022-03-08T18:42:48.113 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../08-python3-prettytable_0.7.2-5_all.deb ... 2022-03-08T18:42:48.149 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2022-03-08T18:42:48.402 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-rbd. 2022-03-08T18:42:48.414 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../09-python3-rbd_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:48.452 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-rbd (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:48.548 INFO:teuthology.orchestra.run.smithi066.stdout:Get:93 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-volume all 17.0.0-10961-g6fe8e680-1focal [219 kB] 2022-03-08T18:42:48.549 INFO:teuthology.orchestra.run.smithi066.stdout:Get:94 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 17.0.0-10961-g6fe8e680-1focal [69.5 kB] 2022-03-08T18:42:48.550 INFO:teuthology.orchestra.run.smithi066.stdout:Get:95 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 17.0.0-10961-g6fe8e680-1focal [21.4 kB] 2022-03-08T18:42:48.550 INFO:teuthology.orchestra.run.smithi066.stdout:Get:96 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 17.0.0-10961-g6fe8e680-1focal [12.7 MB] 2022-03-08T18:42:48.665 INFO:teuthology.orchestra.run.smithi002.stdout:Get:93 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 ceph-volume all 17.0.0-10961-g6fe8e680-1focal [219 kB] 2022-03-08T18:42:48.667 INFO:teuthology.orchestra.run.smithi002.stdout:Get:94 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 17.0.0-10961-g6fe8e680-1focal [69.5 kB] 2022-03-08T18:42:48.668 INFO:teuthology.orchestra.run.smithi002.stdout:Get:95 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 17.0.0-10961-g6fe8e680-1focal [21.4 kB] 2022-03-08T18:42:48.756 INFO:teuthology.orchestra.run.smithi002.stdout:Get:96 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 17.0.0-10961-g6fe8e680-1focal [12.7 MB] 2022-03-08T18:42:48.831 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2022-03-08T18:42:48.845 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../10-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2022-03-08T18:42:48.922 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2022-03-08T18:42:49.115 INFO:teuthology.orchestra.run.smithi066.stdout:Get:97 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 17.0.0-10961-g6fe8e680-1focal [78.8 kB] 2022-03-08T18:42:49.230 INFO:teuthology.orchestra.run.smithi002.stdout:Get:97 https://4.chacra.ceph.com/r/ceph/master/6fe8e680a334072b1ba19b2de006862f98aedbae/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 17.0.0-10961-g6fe8e680-1focal [78.8 kB] 2022-03-08T18:42:49.301 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package librabbitmq4:amd64. 2022-03-08T18:42:49.313 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../11-librabbitmq4_0.10.0-1_amd64.deb ... 2022-03-08T18:42:49.351 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2022-03-08T18:42:49.680 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package librdkafka1:amd64. 2022-03-08T18:42:49.694 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../12-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2022-03-08T18:42:49.729 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2022-03-08T18:42:49.747 INFO:teuthology.orchestra.run.smithi066.stdout:Fetched 181 MB in 17s (10.4 MB/s) 2022-03-08T18:42:49.864 INFO:teuthology.orchestra.run.smithi002.stdout:Fetched 181 MB in 17s (10.4 MB/s) 2022-03-08T18:42:49.932 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2022-03-08T18:42:49.985 INFO:teuthology.orchestra.run.smithi066.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 109337 files and directories currently installed.) 2022-03-08T18:42:49.988 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../00-liblttng-ust0_2.11.0-1_amd64.deb ... 2022-03-08T18:42:50.089 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2022-03-08T18:42:50.101 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2022-03-08T18:42:50.138 INFO:teuthology.orchestra.run.smithi002.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 109337 files and directories currently installed.) 2022-03-08T18:42:50.142 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../00-liblttng-ust0_2.11.0-1_amd64.deb ... 2022-03-08T18:42:50.209 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package librgw2. 2022-03-08T18:42:50.216 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../13-librgw2_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:50.259 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking librgw2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:50.357 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2022-03-08T18:42:50.507 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../01-librbd1_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:50.627 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking librbd1 (17.0.0-10961-g6fe8e680-1focal) over (15.2.14-0ubuntu0.20.04.2) ... 2022-03-08T18:42:50.751 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../01-librbd1_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:50.875 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking librbd1 (17.0.0-10961-g6fe8e680-1focal) over (15.2.14-0ubuntu0.20.04.2) ... 2022-03-08T18:42:50.897 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-rgw. 2022-03-08T18:42:50.911 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../14-python3-rgw_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:50.947 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-rgw (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:51.342 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package liboath0. 2022-03-08T18:42:51.357 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../15-liboath0_2.6.1-1.3_amd64.deb ... 2022-03-08T18:42:51.369 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../02-librados2_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:51.392 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2022-03-08T18:42:51.491 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking librados2 (17.0.0-10961-g6fe8e680-1focal) over (15.2.14-0ubuntu0.20.04.2) ... 2022-03-08T18:42:51.646 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../02-librados2_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:51.770 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking librados2 (17.0.0-10961-g6fe8e680-1focal) over (15.2.14-0ubuntu0.20.04.2) ... 2022-03-08T18:42:52.113 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libradosstriper1. 2022-03-08T18:42:52.125 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../16-libradosstriper1_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:52.163 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libradosstriper1 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:52.296 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libcephfs2. 2022-03-08T18:42:52.309 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../03-libcephfs2_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:52.337 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libcephfs2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:52.608 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libcephfs2. 2022-03-08T18:42:52.620 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../03-libcephfs2_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:52.668 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libcephfs2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:52.677 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph-common. 2022-03-08T18:42:52.689 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../17-ceph-common_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:52.725 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-rados. 2022-03-08T18:42:52.726 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph-common (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:52.734 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../04-python3-rados_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:52.799 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-rados (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:53.093 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-rados. 2022-03-08T18:42:53.107 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../04-python3-rados_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:53.119 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-ceph-argparse. 2022-03-08T18:42:53.132 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../05-python3-ceph-argparse_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:42:53.161 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-rados (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:53.161 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-ceph-argparse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:53.468 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-cephfs. 2022-03-08T18:42:53.479 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../06-python3-cephfs_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:53.530 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-cephfs (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:53.565 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-ceph-argparse. 2022-03-08T18:42:53.573 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../05-python3-ceph-argparse_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:42:53.627 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-ceph-argparse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:53.918 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-ceph-common. 2022-03-08T18:42:53.932 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../07-python3-ceph-common_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:42:53.984 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-ceph-common (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:53.987 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-cephfs. 2022-03-08T18:42:53.999 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../06-python3-cephfs_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:54.037 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-cephfs (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:54.329 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-prettytable. 2022-03-08T18:42:54.345 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../08-python3-prettytable_0.7.2-5_all.deb ... 2022-03-08T18:42:54.388 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2022-03-08T18:42:54.409 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-ceph-common. 2022-03-08T18:42:54.422 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../07-python3-ceph-common_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:42:54.459 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-ceph-common (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:54.666 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-rbd. 2022-03-08T18:42:54.683 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../09-python3-rbd_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:54.754 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-rbd (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:54.874 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-prettytable. 2022-03-08T18:42:54.889 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../08-python3-prettytable_0.7.2-5_all.deb ... 2022-03-08T18:42:54.938 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2022-03-08T18:42:55.145 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2022-03-08T18:42:55.154 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../10-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2022-03-08T18:42:55.194 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2022-03-08T18:42:55.244 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-rbd. 2022-03-08T18:42:55.256 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../09-python3-rbd_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:55.294 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-rbd (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:55.385 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph-base. 2022-03-08T18:42:55.399 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../18-ceph-base_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:55.463 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph-base (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:55.532 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package librabbitmq4:amd64. 2022-03-08T18:42:55.541 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../11-librabbitmq4_0.10.0-1_amd64.deb ... 2022-03-08T18:42:55.574 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2022-03-08T18:42:55.694 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2022-03-08T18:42:55.703 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../10-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2022-03-08T18:42:55.768 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2022-03-08T18:42:55.986 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package librdkafka1:amd64. 2022-03-08T18:42:56.001 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../12-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2022-03-08T18:42:56.036 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2022-03-08T18:42:56.402 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package librabbitmq4:amd64. 2022-03-08T18:42:56.411 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../11-librabbitmq4_0.10.0-1_amd64.deb ... 2022-03-08T18:42:56.453 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2022-03-08T18:42:56.490 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package librgw2. 2022-03-08T18:42:56.505 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../13-librgw2_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:56.534 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-cryptography. 2022-03-08T18:42:56.544 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../19-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2022-03-08T18:42:56.557 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking librgw2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:56.583 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2022-03-08T18:42:56.799 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package librdkafka1:amd64. 2022-03-08T18:42:56.815 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../12-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2022-03-08T18:42:56.858 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2022-03-08T18:42:56.971 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-openssl. 2022-03-08T18:42:56.982 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../20-python3-openssl_19.0.0-1build1_all.deb ... 2022-03-08T18:42:57.021 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2022-03-08T18:42:57.308 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2022-03-08T18:42:57.321 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../21-ceph-mgr-modules-core_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:42:57.366 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph-mgr-modules-core (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:57.376 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package librgw2. 2022-03-08T18:42:57.386 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../13-librgw2_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:57.427 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking librgw2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:57.485 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-rgw. 2022-03-08T18:42:57.498 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../14-python3-rgw_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:57.535 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-rgw (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:57.738 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-bcrypt. 2022-03-08T18:42:57.750 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../22-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2022-03-08T18:42:57.788 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2022-03-08T18:42:57.855 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package liboath0. 2022-03-08T18:42:57.868 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../15-liboath0_2.6.1-1.3_amd64.deb ... 2022-03-08T18:42:57.946 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2022-03-08T18:42:58.091 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-cherrypy3. 2022-03-08T18:42:58.103 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../23-python3-cherrypy3_8.9.1-6_all.deb ... 2022-03-08T18:42:58.141 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2022-03-08T18:42:58.172 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-rgw. 2022-03-08T18:42:58.186 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../14-python3-rgw_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:58.233 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-rgw (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:58.241 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libradosstriper1. 2022-03-08T18:42:58.255 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../16-libradosstriper1_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:58.300 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libradosstriper1 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:58.579 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-jwt. 2022-03-08T18:42:58.586 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../24-python3-jwt_1.7.1-2ubuntu2_all.deb ... 2022-03-08T18:42:58.628 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2) ... 2022-03-08T18:42:58.770 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package liboath0. 2022-03-08T18:42:58.779 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../15-liboath0_2.6.1-1.3_amd64.deb ... 2022-03-08T18:42:58.821 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph-common. 2022-03-08T18:42:58.829 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2022-03-08T18:42:58.833 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../17-ceph-common_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:58.871 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph-common (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:59.016 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-logutils. 2022-03-08T18:42:59.093 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../25-python3-logutils_0.3.3-7_all.deb ... 2022-03-08T18:42:59.093 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-logutils (0.3.3-7) ... 2022-03-08T18:42:59.286 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libradosstriper1. 2022-03-08T18:42:59.302 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../16-libradosstriper1_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:59.352 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libradosstriper1 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:42:59.370 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-markupsafe. 2022-03-08T18:42:59.383 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../26-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2022-03-08T18:42:59.436 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2022-03-08T18:42:59.857 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-mako. 2022-03-08T18:42:59.868 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../27-python3-mako_1.1.0+ds1-1ubuntu2_all.deb ... 2022-03-08T18:42:59.907 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2) ... 2022-03-08T18:42:59.935 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph-common. 2022-03-08T18:42:59.949 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../17-ceph-common_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:42:59.995 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph-common (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:00.227 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-simplegeneric. 2022-03-08T18:43:00.239 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../28-python3-simplegeneric_0.8.1-3_all.deb ... 2022-03-08T18:43:00.277 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2022-03-08T18:43:00.555 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-singledispatch. 2022-03-08T18:43:00.570 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../29-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2022-03-08T18:43:00.613 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2022-03-08T18:43:00.976 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-webob. 2022-03-08T18:43:00.989 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../30-python3-webob_1%3a1.8.5-2_all.deb ... 2022-03-08T18:43:01.034 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2022-03-08T18:43:01.396 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-soupsieve. 2022-03-08T18:43:01.410 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../31-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2022-03-08T18:43:01.463 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2022-03-08T18:43:01.463 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph-base. 2022-03-08T18:43:01.478 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../18-ceph-base_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:01.557 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph-base (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:01.758 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-bs4. 2022-03-08T18:43:01.773 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../32-python3-bs4_4.8.2-1_all.deb ... 2022-03-08T18:43:01.816 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2022-03-08T18:43:02.784 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph-base. 2022-03-08T18:43:02.801 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../18-ceph-base_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:02.848 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph-base (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:03.063 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-waitress. 2022-03-08T18:43:03.079 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../33-python3-waitress_1.4.1-1_all.deb ... 2022-03-08T18:43:03.150 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-waitress (1.4.1-1) ... 2022-03-08T18:43:03.484 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-tempita. 2022-03-08T18:43:03.498 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../34-python3-tempita_0.5.2-6_all.deb ... 2022-03-08T18:43:03.520 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-cryptography. 2022-03-08T18:43:03.530 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../19-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2022-03-08T18:43:03.550 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-tempita (0.5.2-6) ... 2022-03-08T18:43:03.570 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2022-03-08T18:43:03.929 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-paste. 2022-03-08T18:43:03.943 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../35-python3-paste_3.4.0+dfsg1-1_all.deb ... 2022-03-08T18:43:03.996 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2022-03-08T18:43:04.076 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-openssl. 2022-03-08T18:43:04.090 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../20-python3-openssl_19.0.0-1build1_all.deb ... 2022-03-08T18:43:04.134 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2022-03-08T18:43:04.384 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2022-03-08T18:43:04.397 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../36-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2022-03-08T18:43:04.442 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2022-03-08T18:43:04.530 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2022-03-08T18:43:04.538 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../21-ceph-mgr-modules-core_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:04.587 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph-mgr-modules-core (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:04.687 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-pastedeploy. 2022-03-08T18:43:04.702 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../37-python3-pastedeploy_2.1.0-1_all.deb ... 2022-03-08T18:43:04.736 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2022-03-08T18:43:04.985 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-bcrypt. 2022-03-08T18:43:04.996 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../22-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2022-03-08T18:43:05.035 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2022-03-08T18:43:05.040 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-webtest. 2022-03-08T18:43:05.053 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../38-python3-webtest_2.0.32-2_all.deb ... 2022-03-08T18:43:05.098 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-webtest (2.0.32-2) ... 2022-03-08T18:43:05.389 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-cherrypy3. 2022-03-08T18:43:05.400 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../23-python3-cherrypy3_8.9.1-6_all.deb ... 2022-03-08T18:43:05.447 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2022-03-08T18:43:05.462 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-pecan. 2022-03-08T18:43:05.473 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../39-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2022-03-08T18:43:05.540 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2022-03-08T18:43:05.578 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-cryptography. 2022-03-08T18:43:05.594 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../19-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2022-03-08T18:43:05.624 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2022-03-08T18:43:05.851 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-jwt. 2022-03-08T18:43:05.866 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../24-python3-jwt_1.7.1-2ubuntu2_all.deb ... 2022-03-08T18:43:05.909 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2) ... 2022-03-08T18:43:05.923 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-werkzeug. 2022-03-08T18:43:05.937 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../40-python3-werkzeug_0.16.1+dfsg1-2_all.deb ... 2022-03-08T18:43:05.981 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2) ... 2022-03-08T18:43:06.101 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-openssl. 2022-03-08T18:43:06.114 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../20-python3-openssl_19.0.0-1build1_all.deb ... 2022-03-08T18:43:06.164 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2022-03-08T18:43:06.221 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-logutils. 2022-03-08T18:43:06.236 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../25-python3-logutils_0.3.3-7_all.deb ... 2022-03-08T18:43:06.305 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-logutils (0.3.3-7) ... 2022-03-08T18:43:06.368 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2022-03-08T18:43:06.383 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../41-libsqlite3-mod-ceph_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:06.441 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libsqlite3-mod-ceph (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:06.519 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2022-03-08T18:43:06.534 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../21-ceph-mgr-modules-core_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:06.590 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph-mgr-modules-core (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:06.758 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-markupsafe. 2022-03-08T18:43:06.771 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../26-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2022-03-08T18:43:06.842 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2022-03-08T18:43:06.897 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph-mgr. 2022-03-08T18:43:06.910 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../42-ceph-mgr_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:06.966 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph-mgr (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:07.040 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-bcrypt. 2022-03-08T18:43:07.055 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../22-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2022-03-08T18:43:07.090 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2022-03-08T18:43:07.121 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-mako. 2022-03-08T18:43:07.135 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../27-python3-mako_1.1.0+ds1-1ubuntu2_all.deb ... 2022-03-08T18:43:07.178 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2) ... 2022-03-08T18:43:07.434 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph-mon. 2022-03-08T18:43:07.447 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../43-ceph-mon_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:07.484 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph-mon (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:07.497 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-cherrypy3. 2022-03-08T18:43:07.509 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../23-python3-cherrypy3_8.9.1-6_all.deb ... 2022-03-08T18:43:07.547 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2022-03-08T18:43:07.557 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-simplegeneric. 2022-03-08T18:43:07.572 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../28-python3-simplegeneric_0.8.1-3_all.deb ... 2022-03-08T18:43:07.624 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2022-03-08T18:43:08.059 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-jwt. 2022-03-08T18:43:08.062 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-singledispatch. 2022-03-08T18:43:08.075 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../29-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2022-03-08T18:43:08.077 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../24-python3-jwt_1.7.1-2ubuntu2_all.deb ... 2022-03-08T18:43:08.113 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2022-03-08T18:43:08.116 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2) ... 2022-03-08T18:43:08.432 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-webob. 2022-03-08T18:43:08.445 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-logutils. 2022-03-08T18:43:08.446 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../30-python3-webob_1%3a1.8.5-2_all.deb ... 2022-03-08T18:43:08.461 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../25-python3-logutils_0.3.3-7_all.deb ... 2022-03-08T18:43:08.489 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2022-03-08T18:43:08.503 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-logutils (0.3.3-7) ... 2022-03-08T18:43:08.648 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph-osd. 2022-03-08T18:43:08.660 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../44-ceph-osd_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:08.698 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph-osd (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:08.777 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-soupsieve. 2022-03-08T18:43:08.792 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../31-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2022-03-08T18:43:08.835 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2022-03-08T18:43:08.892 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-markupsafe. 2022-03-08T18:43:08.906 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../26-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2022-03-08T18:43:08.947 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2022-03-08T18:43:09.217 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-bs4. 2022-03-08T18:43:09.234 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../32-python3-bs4_4.8.2-1_all.deb ... 2022-03-08T18:43:09.314 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2022-03-08T18:43:09.413 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-mako. 2022-03-08T18:43:09.427 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../27-python3-mako_1.1.0+ds1-1ubuntu2_all.deb ... 2022-03-08T18:43:09.496 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2) ... 2022-03-08T18:43:09.705 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-waitress. 2022-03-08T18:43:09.720 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../33-python3-waitress_1.4.1-1_all.deb ... 2022-03-08T18:43:09.766 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-waitress (1.4.1-1) ... 2022-03-08T18:43:09.791 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-simplegeneric. 2022-03-08T18:43:09.803 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../28-python3-simplegeneric_0.8.1-3_all.deb ... 2022-03-08T18:43:09.840 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2022-03-08T18:43:10.061 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-tempita. 2022-03-08T18:43:10.079 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../34-python3-tempita_0.5.2-6_all.deb ... 2022-03-08T18:43:10.111 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-tempita (0.5.2-6) ... 2022-03-08T18:43:10.191 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-singledispatch. 2022-03-08T18:43:10.206 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../29-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2022-03-08T18:43:10.253 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2022-03-08T18:43:10.448 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-paste. 2022-03-08T18:43:10.456 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../35-python3-paste_3.4.0+dfsg1-1_all.deb ... 2022-03-08T18:43:10.498 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2022-03-08T18:43:10.722 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-webob. 2022-03-08T18:43:10.736 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../30-python3-webob_1%3a1.8.5-2_all.deb ... 2022-03-08T18:43:10.782 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2022-03-08T18:43:10.785 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2022-03-08T18:43:10.800 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../36-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2022-03-08T18:43:10.835 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2022-03-08T18:43:11.154 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-pastedeploy. 2022-03-08T18:43:11.168 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../37-python3-pastedeploy_2.1.0-1_all.deb ... 2022-03-08T18:43:11.204 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2022-03-08T18:43:11.262 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-soupsieve. 2022-03-08T18:43:11.277 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../31-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2022-03-08T18:43:11.321 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2022-03-08T18:43:11.575 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-webtest. 2022-03-08T18:43:11.588 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../38-python3-webtest_2.0.32-2_all.deb ... 2022-03-08T18:43:11.596 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph. 2022-03-08T18:43:11.613 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../45-ceph_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:11.634 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-webtest (2.0.32-2) ... 2022-03-08T18:43:11.654 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:11.716 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-bs4. 2022-03-08T18:43:11.731 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../32-python3-bs4_4.8.2-1_all.deb ... 2022-03-08T18:43:11.769 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2022-03-08T18:43:11.970 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-pecan. 2022-03-08T18:43:11.979 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../39-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2022-03-08T18:43:12.008 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph-fuse. 2022-03-08T18:43:12.015 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../46-ceph-fuse_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:12.019 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2022-03-08T18:43:12.058 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph-fuse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:12.148 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-waitress. 2022-03-08T18:43:12.160 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../33-python3-waitress_1.4.1-1_all.deb ... 2022-03-08T18:43:12.200 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-waitress (1.4.1-1) ... 2022-03-08T18:43:12.386 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-werkzeug. 2022-03-08T18:43:12.397 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../40-python3-werkzeug_0.16.1+dfsg1-2_all.deb ... 2022-03-08T18:43:12.411 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph-mds. 2022-03-08T18:43:12.421 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../47-ceph-mds_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:12.466 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2) ... 2022-03-08T18:43:12.478 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph-mds (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:12.672 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-tempita. 2022-03-08T18:43:12.687 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../34-python3-tempita_0.5.2-6_all.deb ... 2022-03-08T18:43:12.730 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-tempita (0.5.2-6) ... 2022-03-08T18:43:12.889 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2022-03-08T18:43:12.902 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../41-libsqlite3-mod-ceph_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:12.963 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libsqlite3-mod-ceph (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:13.008 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-repoze.lru. 2022-03-08T18:43:13.022 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../48-python3-repoze.lru_0.7-2_all.deb ... 2022-03-08T18:43:13.034 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-paste. 2022-03-08T18:43:13.047 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../35-python3-paste_3.4.0+dfsg1-1_all.deb ... 2022-03-08T18:43:13.058 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2022-03-08T18:43:13.091 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2022-03-08T18:43:13.318 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph-mgr. 2022-03-08T18:43:13.334 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../42-ceph-mgr_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:13.336 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libjs-underscore. 2022-03-08T18:43:13.350 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../49-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2022-03-08T18:43:13.375 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph-mgr (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:13.411 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2022-03-08T18:43:13.524 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2022-03-08T18:43:13.535 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../36-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2022-03-08T18:43:13.586 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2022-03-08T18:43:13.798 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libjs-sphinxdoc. 2022-03-08T18:43:13.811 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../50-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2022-03-08T18:43:13.852 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2022-03-08T18:43:14.039 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph-mon. 2022-03-08T18:43:14.052 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../43-ceph-mon_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:14.106 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph-mon (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:14.143 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-pastedeploy. 2022-03-08T18:43:14.155 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../37-python3-pastedeploy_2.1.0-1_all.deb ... 2022-03-08T18:43:14.195 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2022-03-08T18:43:14.210 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-routes. 2022-03-08T18:43:14.223 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../51-python3-routes_2.4.1-2_all.deb ... 2022-03-08T18:43:14.260 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-routes (2.4.1-2) ... 2022-03-08T18:43:14.520 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-webtest. 2022-03-08T18:43:14.535 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../38-python3-webtest_2.0.32-2_all.deb ... 2022-03-08T18:43:14.578 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-webtest (2.0.32-2) ... 2022-03-08T18:43:14.607 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2022-03-08T18:43:14.615 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../52-ceph-mgr-dashboard_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:14.671 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph-mgr-dashboard (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:15.070 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-pecan. 2022-03-08T18:43:15.084 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../39-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2022-03-08T18:43:15.148 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2022-03-08T18:43:15.236 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-decorator. 2022-03-08T18:43:15.250 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../53-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2022-03-08T18:43:15.294 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2022-03-08T18:43:15.478 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph-osd. 2022-03-08T18:43:15.487 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../44-ceph-osd_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:15.553 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph-osd (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:15.579 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2022-03-08T18:43:15.590 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-werkzeug. 2022-03-08T18:43:15.591 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../54-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2022-03-08T18:43:15.606 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../40-python3-werkzeug_0.16.1+dfsg1-2_all.deb ... 2022-03-08T18:43:15.620 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2022-03-08T18:43:15.652 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2) ... 2022-03-08T18:43:15.949 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-scipy. 2022-03-08T18:43:15.963 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../55-python3-scipy_1.3.3-3build1_amd64.deb ... 2022-03-08T18:43:16.007 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2022-03-08T18:43:16.157 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2022-03-08T18:43:16.172 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../41-libsqlite3-mod-ceph_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:16.207 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libsqlite3-mod-ceph (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:16.606 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph-mgr. 2022-03-08T18:43:16.619 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../42-ceph-mgr_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:16.655 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph-mgr (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:17.250 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph-mon. 2022-03-08T18:43:17.264 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../43-ceph-mon_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:17.325 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph-mon (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:17.401 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-sklearn-lib. 2022-03-08T18:43:17.415 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../56-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2022-03-08T18:43:17.458 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2022-03-08T18:43:18.029 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-joblib. 2022-03-08T18:43:18.043 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../57-python3-joblib_0.14.0-3_all.deb ... 2022-03-08T18:43:18.121 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-joblib (0.14.0-3) ... 2022-03-08T18:43:18.451 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph. 2022-03-08T18:43:18.468 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../45-ceph_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:18.509 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:18.626 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-sklearn. 2022-03-08T18:43:18.630 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph-osd. 2022-03-08T18:43:18.639 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../58-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2022-03-08T18:43:18.645 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../44-ceph-osd_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:18.676 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2022-03-08T18:43:18.708 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph-osd (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:18.812 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph-fuse. 2022-03-08T18:43:18.827 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../46-ceph-fuse_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:18.870 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph-fuse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:19.409 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph-mds. 2022-03-08T18:43:19.419 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../47-ceph-mds_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:19.494 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph-mds (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:19.637 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2022-03-08T18:43:19.653 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../59-ceph-mgr-diskprediction-local_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:19.695 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph-mgr-diskprediction-local (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:20.056 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-repoze.lru. 2022-03-08T18:43:20.069 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../48-python3-repoze.lru_0.7-2_all.deb ... 2022-03-08T18:43:20.116 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2022-03-08T18:43:20.439 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libjs-underscore. 2022-03-08T18:43:20.452 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../49-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2022-03-08T18:43:20.488 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2022-03-08T18:43:20.791 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libjs-sphinxdoc. 2022-03-08T18:43:20.804 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../50-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2022-03-08T18:43:20.850 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2022-03-08T18:43:20.908 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-cachetools. 2022-03-08T18:43:20.921 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../60-python3-cachetools_4.0.0-1_all.deb ... 2022-03-08T18:43:20.966 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2022-03-08T18:43:21.229 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-routes. 2022-03-08T18:43:21.243 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../51-python3-routes_2.4.1-2_all.deb ... 2022-03-08T18:43:21.287 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-routes (2.4.1-2) ... 2022-03-08T18:43:21.303 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-pyasn1. 2022-03-08T18:43:21.315 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../61-python3-pyasn1_0.4.2-3build1_all.deb ... 2022-03-08T18:43:21.361 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2022-03-08T18:43:21.591 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2022-03-08T18:43:21.606 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../52-ceph-mgr-dashboard_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:21.649 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph-mgr-dashboard (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:21.690 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-pyasn1-modules. 2022-03-08T18:43:21.699 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../62-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2022-03-08T18:43:21.740 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2022-03-08T18:43:21.856 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph. 2022-03-08T18:43:21.870 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../45-ceph_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:21.914 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:22.195 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-rsa. 2022-03-08T18:43:22.210 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../63-python3-rsa_4.0-3_all.deb ... 2022-03-08T18:43:22.253 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-rsa (4.0-3) ... 2022-03-08T18:43:22.281 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-decorator. 2022-03-08T18:43:22.295 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../53-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2022-03-08T18:43:22.339 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2022-03-08T18:43:22.382 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph-fuse. 2022-03-08T18:43:22.390 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../46-ceph-fuse_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:22.477 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph-fuse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:22.582 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-google-auth. 2022-03-08T18:43:22.595 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../64-python3-google-auth_1.5.1-2_all.deb ... 2022-03-08T18:43:22.632 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2022-03-08T18:43:22.635 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2022-03-08T18:43:22.647 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../54-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2022-03-08T18:43:22.684 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2022-03-08T18:43:22.911 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph-mds. 2022-03-08T18:43:22.923 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../47-ceph-mds_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:22.978 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph-mds (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:22.978 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-blinker. 2022-03-08T18:43:22.988 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-scipy. 2022-03-08T18:43:22.991 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../65-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2022-03-08T18:43:23.000 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../55-python3-scipy_1.3.3-3build1_amd64.deb ... 2022-03-08T18:43:23.029 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2022-03-08T18:43:23.036 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2022-03-08T18:43:23.448 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-oauthlib. 2022-03-08T18:43:23.462 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../66-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2022-03-08T18:43:23.515 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2022-03-08T18:43:23.648 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-repoze.lru. 2022-03-08T18:43:23.663 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../48-python3-repoze.lru_0.7-2_all.deb ... 2022-03-08T18:43:23.722 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2022-03-08T18:43:24.101 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libjs-underscore. 2022-03-08T18:43:24.114 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../49-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2022-03-08T18:43:24.147 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2022-03-08T18:43:24.544 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libjs-sphinxdoc. 2022-03-08T18:43:24.557 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../50-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2022-03-08T18:43:24.633 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2022-03-08T18:43:24.709 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-sklearn-lib. 2022-03-08T18:43:24.721 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../56-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2022-03-08T18:43:24.758 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2022-03-08T18:43:24.779 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-requests-oauthlib. 2022-03-08T18:43:24.794 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../67-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2022-03-08T18:43:24.845 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2022-03-08T18:43:24.981 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-routes. 2022-03-08T18:43:24.989 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../51-python3-routes_2.4.1-2_all.deb ... 2022-03-08T18:43:25.087 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-routes (2.4.1-2) ... 2022-03-08T18:43:25.209 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-websocket. 2022-03-08T18:43:25.225 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../68-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2022-03-08T18:43:25.275 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2022-03-08T18:43:25.355 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-joblib. 2022-03-08T18:43:25.368 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../57-python3-joblib_0.14.0-3_all.deb ... 2022-03-08T18:43:25.412 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-joblib (0.14.0-3) ... 2022-03-08T18:43:25.432 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2022-03-08T18:43:25.448 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../52-ceph-mgr-dashboard_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:25.490 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph-mgr-dashboard (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:25.629 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-kubernetes. 2022-03-08T18:43:25.642 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../69-python3-kubernetes_7.0.0~a1-2_all.deb ... 2022-03-08T18:43:25.687 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2022-03-08T18:43:25.842 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-sklearn. 2022-03-08T18:43:25.857 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../58-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2022-03-08T18:43:25.900 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2022-03-08T18:43:26.240 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-decorator. 2022-03-08T18:43:26.253 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../53-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2022-03-08T18:43:26.315 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2022-03-08T18:43:26.320 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2022-03-08T18:43:26.333 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../70-ceph-mgr-k8sevents_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:26.378 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph-mgr-k8sevents (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:26.625 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2022-03-08T18:43:26.627 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2022-03-08T18:43:26.638 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../54-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2022-03-08T18:43:26.641 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../59-ceph-mgr-diskprediction-local_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:26.675 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2022-03-08T18:43:26.676 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph-mgr-diskprediction-local (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:26.698 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libonig5:amd64. 2022-03-08T18:43:26.712 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../71-libonig5_6.9.4-1_amd64.deb ... 2022-03-08T18:43:26.764 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2022-03-08T18:43:27.061 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-scipy. 2022-03-08T18:43:27.068 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../55-python3-scipy_1.3.3-3build1_amd64.deb ... 2022-03-08T18:43:27.119 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2022-03-08T18:43:27.168 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libjq1:amd64. 2022-03-08T18:43:27.183 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../72-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2022-03-08T18:43:27.226 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2022-03-08T18:43:27.580 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package jq. 2022-03-08T18:43:27.593 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../73-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2022-03-08T18:43:27.646 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2022-03-08T18:43:28.017 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-cachetools. 2022-03-08T18:43:28.030 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../60-python3-cachetools_4.0.0-1_all.deb ... 2022-03-08T18:43:28.042 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libxslt1.1:amd64. 2022-03-08T18:43:28.058 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../74-libxslt1.1_1.1.34-4_amd64.deb ... 2022-03-08T18:43:28.074 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2022-03-08T18:43:28.100 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libxslt1.1:amd64 (1.1.34-4) ... 2022-03-08T18:43:28.395 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package xmlstarlet. 2022-03-08T18:43:28.406 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../75-xmlstarlet_1.6.1-2build1_amd64.deb ... 2022-03-08T18:43:28.411 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-pyasn1. 2022-03-08T18:43:28.428 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../61-python3-pyasn1_0.4.2-3build1_all.deb ... 2022-03-08T18:43:28.445 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2022-03-08T18:43:28.469 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2022-03-08T18:43:28.833 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-pyasn1-modules. 2022-03-08T18:43:28.846 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../62-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2022-03-08T18:43:28.917 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph-test. 2022-03-08T18:43:28.932 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../76-ceph-test_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:28.984 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph-test (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:29.170 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-sklearn-lib. 2022-03-08T18:43:29.184 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../56-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2022-03-08T18:43:29.236 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2022-03-08T18:43:29.303 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2022-03-08T18:43:29.958 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-joblib. 2022-03-08T18:43:29.972 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../57-python3-joblib_0.14.0-3_all.deb ... 2022-03-08T18:43:30.025 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-joblib (0.14.0-3) ... 2022-03-08T18:43:30.054 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-rsa. 2022-03-08T18:43:30.067 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../63-python3-rsa_4.0-3_all.deb ... 2022-03-08T18:43:30.111 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-rsa (4.0-3) ... 2022-03-08T18:43:30.558 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-sklearn. 2022-03-08T18:43:30.571 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../58-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2022-03-08T18:43:30.600 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-google-auth. 2022-03-08T18:43:30.613 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../64-python3-google-auth_1.5.1-2_all.deb ... 2022-03-08T18:43:30.640 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2022-03-08T18:43:30.657 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2022-03-08T18:43:30.987 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-blinker. 2022-03-08T18:43:31.001 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../65-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2022-03-08T18:43:31.078 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2022-03-08T18:43:31.440 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-oauthlib. 2022-03-08T18:43:31.456 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../66-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2022-03-08T18:43:31.507 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2022-03-08T18:43:31.546 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2022-03-08T18:43:31.560 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../59-ceph-mgr-diskprediction-local_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:31.612 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph-mgr-diskprediction-local (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:31.903 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-requests-oauthlib. 2022-03-08T18:43:31.918 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../67-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2022-03-08T18:43:31.969 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2022-03-08T18:43:32.373 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-websocket. 2022-03-08T18:43:32.387 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../68-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2022-03-08T18:43:32.431 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2022-03-08T18:43:32.840 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-kubernetes. 2022-03-08T18:43:32.855 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../69-python3-kubernetes_7.0.0~a1-2_all.deb ... 2022-03-08T18:43:32.918 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2022-03-08T18:43:33.055 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-cachetools. 2022-03-08T18:43:33.070 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../60-python3-cachetools_4.0.0-1_all.deb ... 2022-03-08T18:43:33.117 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2022-03-08T18:43:33.482 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-pyasn1. 2022-03-08T18:43:33.494 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../61-python3-pyasn1_0.4.2-3build1_all.deb ... 2022-03-08T18:43:33.499 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2022-03-08T18:43:33.510 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../70-ceph-mgr-k8sevents_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:33.533 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2022-03-08T18:43:33.549 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph-mgr-k8sevents (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:33.852 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package ceph-volume. 2022-03-08T18:43:33.864 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../77-ceph-volume_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:33.919 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking ceph-volume (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:33.979 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libonig5:amd64. 2022-03-08T18:43:33.993 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../71-libonig5_6.9.4-1_amd64.deb ... 2022-03-08T18:43:34.062 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2022-03-08T18:43:34.080 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-pyasn1-modules. 2022-03-08T18:43:34.093 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../62-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2022-03-08T18:43:34.206 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2022-03-08T18:43:34.367 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package cephadm. 2022-03-08T18:43:34.380 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../78-cephadm_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:34.466 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking cephadm (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:34.525 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libjq1:amd64. 2022-03-08T18:43:34.540 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../72-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2022-03-08T18:43:34.624 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-rsa. 2022-03-08T18:43:34.638 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../63-python3-rsa_4.0-3_all.deb ... 2022-03-08T18:43:34.699 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-rsa (4.0-3) ... 2022-03-08T18:43:34.708 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2022-03-08T18:43:34.787 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package libcephfs-dev. 2022-03-08T18:43:34.802 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../79-libcephfs-dev_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:34.844 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking libcephfs-dev (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:35.011 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-google-auth. 2022-03-08T18:43:35.020 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package jq. 2022-03-08T18:43:35.023 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../64-python3-google-auth_1.5.1-2_all.deb ... 2022-03-08T18:43:35.036 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../73-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2022-03-08T18:43:35.086 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2022-03-08T18:43:35.087 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2022-03-08T18:43:35.215 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package nvme-cli. 2022-03-08T18:43:35.228 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../80-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2022-03-08T18:43:35.287 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2022-03-08T18:43:35.462 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-blinker. 2022-03-08T18:43:35.473 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libxslt1.1:amd64. 2022-03-08T18:43:35.475 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../65-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2022-03-08T18:43:35.487 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../74-libxslt1.1_1.1.34-4_amd64.deb ... 2022-03-08T18:43:35.531 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libxslt1.1:amd64 (1.1.34-4) ... 2022-03-08T18:43:35.553 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2022-03-08T18:43:35.898 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-atomicwrites. 2022-03-08T18:43:35.912 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../81-python3-atomicwrites_1.1.5-2build1_all.deb ... 2022-03-08T18:43:35.918 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package xmlstarlet. 2022-03-08T18:43:35.933 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../75-xmlstarlet_1.6.1-2build1_amd64.deb ... 2022-03-08T18:43:36.005 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2022-03-08T18:43:36.014 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2022-03-08T18:43:36.334 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-attr. 2022-03-08T18:43:36.350 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../82-python3-attr_19.3.0-2_all.deb ... 2022-03-08T18:43:36.392 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-attr (19.3.0-2) ... 2022-03-08T18:43:36.399 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph-test. 2022-03-08T18:43:36.413 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../76-ceph-test_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:36.457 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph-test (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:36.730 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-webencodings. 2022-03-08T18:43:36.739 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../83-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2022-03-08T18:43:36.787 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2022-03-08T18:43:36.800 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-oauthlib. 2022-03-08T18:43:36.815 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../66-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2022-03-08T18:43:36.875 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2022-03-08T18:43:37.191 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-html5lib. 2022-03-08T18:43:37.204 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../84-python3-html5lib_1.0.1-2_all.deb ... 2022-03-08T18:43:37.249 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2022-03-08T18:43:37.458 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-requests-oauthlib. 2022-03-08T18:43:37.474 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../67-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2022-03-08T18:43:37.526 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2022-03-08T18:43:37.654 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-lxml:amd64. 2022-03-08T18:43:37.670 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../85-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2022-03-08T18:43:37.720 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2022-03-08T18:43:37.947 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-websocket. 2022-03-08T18:43:37.960 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../68-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2022-03-08T18:43:38.013 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2022-03-08T18:43:38.208 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-packaging. 2022-03-08T18:43:38.224 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../86-python3-packaging_20.3-1_all.deb ... 2022-03-08T18:43:38.274 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-packaging (20.3-1) ... 2022-03-08T18:43:38.405 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-kubernetes. 2022-03-08T18:43:38.419 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../69-python3-kubernetes_7.0.0~a1-2_all.deb ... 2022-03-08T18:43:38.505 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2022-03-08T18:43:38.712 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-pastescript. 2022-03-08T18:43:38.729 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../87-python3-pastescript_2.0.2-4_all.deb ... 2022-03-08T18:43:38.770 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2022-03-08T18:43:39.125 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-pluggy. 2022-03-08T18:43:39.141 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../88-python3-pluggy_0.13.0-2_all.deb ... 2022-03-08T18:43:39.191 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2022-03-08T18:43:39.286 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2022-03-08T18:43:39.300 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../70-ceph-mgr-k8sevents_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:39.369 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph-mgr-k8sevents (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:39.562 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-psutil. 2022-03-08T18:43:39.577 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../89-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2022-03-08T18:43:39.620 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2022-03-08T18:43:39.736 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libonig5:amd64. 2022-03-08T18:43:39.746 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../71-libonig5_6.9.4-1_amd64.deb ... 2022-03-08T18:43:39.821 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2022-03-08T18:43:40.074 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-py. 2022-03-08T18:43:40.088 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../90-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2022-03-08T18:43:40.166 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2022-03-08T18:43:40.260 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libjq1:amd64. 2022-03-08T18:43:40.275 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../72-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2022-03-08T18:43:40.317 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2022-03-08T18:43:40.512 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-pyinotify. 2022-03-08T18:43:40.525 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../91-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2022-03-08T18:43:40.578 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2022-03-08T18:43:40.587 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package jq. 2022-03-08T18:43:40.596 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../73-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2022-03-08T18:43:40.637 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2022-03-08T18:43:40.907 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-wcwidth. 2022-03-08T18:43:40.920 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../92-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2022-03-08T18:43:40.973 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2022-03-08T18:43:41.079 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libxslt1.1:amd64. 2022-03-08T18:43:41.091 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../74-libxslt1.1_1.1.34-4_amd64.deb ... 2022-03-08T18:43:41.137 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libxslt1.1:amd64 (1.1.34-4) ... 2022-03-08T18:43:41.427 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package ceph-volume. 2022-03-08T18:43:41.427 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-pytest. 2022-03-08T18:43:41.441 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../77-ceph-volume_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:41.442 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../93-python3-pytest_4.6.9-1_all.deb ... 2022-03-08T18:43:41.494 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-pytest (4.6.9-1) ... 2022-03-08T18:43:41.531 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking ceph-volume (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:41.700 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package xmlstarlet. 2022-03-08T18:43:41.712 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../75-xmlstarlet_1.6.1-2build1_amd64.deb ... 2022-03-08T18:43:41.759 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2022-03-08T18:43:41.848 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package python3-simplejson. 2022-03-08T18:43:41.865 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../94-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2022-03-08T18:43:41.882 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package cephadm. 2022-03-08T18:43:41.893 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../78-cephadm_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:41.906 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2022-03-08T18:43:41.931 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking cephadm (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:42.162 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph-test. 2022-03-08T18:43:42.171 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../76-ceph-test_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:42.210 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package libcephfs-dev. 2022-03-08T18:43:42.211 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph-test (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:42.224 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../79-libcephfs-dev_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:42.267 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking libcephfs-dev (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:42.394 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package radosgw. 2022-03-08T18:43:42.409 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../95-radosgw_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:42.452 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking radosgw (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:42.798 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package nvme-cli. 2022-03-08T18:43:42.813 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../80-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2022-03-08T18:43:42.886 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2022-03-08T18:43:43.413 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-atomicwrites. 2022-03-08T18:43:43.425 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../81-python3-atomicwrites_1.1.5-2build1_all.deb ... 2022-03-08T18:43:43.462 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2022-03-08T18:43:43.950 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-attr. 2022-03-08T18:43:43.964 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../82-python3-attr_19.3.0-2_all.deb ... 2022-03-08T18:43:43.991 INFO:teuthology.orchestra.run.smithi137.stdout:Selecting previously unselected package rbd-fuse. 2022-03-08T18:43:44.006 INFO:teuthology.orchestra.run.smithi137.stdout:Preparing to unpack .../96-rbd-fuse_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:44.017 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-attr (19.3.0-2) ... 2022-03-08T18:43:44.041 INFO:teuthology.orchestra.run.smithi137.stdout:Unpacking rbd-fuse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:44.335 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-attr (19.3.0-2) ... 2022-03-08T18:43:44.429 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-webencodings. 2022-03-08T18:43:44.442 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../83-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2022-03-08T18:43:44.479 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2022-03-08T18:43:44.681 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2022-03-08T18:43:44.799 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2022-03-08T18:43:44.849 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-html5lib. 2022-03-08T18:43:44.865 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../84-python3-html5lib_1.0.1-2_all.deb ... 2022-03-08T18:43:44.907 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2022-03-08T18:43:44.985 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up cephadm (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:45.371 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-lxml:amd64. 2022-03-08T18:43:45.385 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../85-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2022-03-08T18:43:45.428 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2022-03-08T18:43:45.647 INFO:teuthology.orchestra.run.smithi137.stdout:Adding system user cephadm....done 2022-03-08T18:43:45.842 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-waitress (1.4.1-1) ... 2022-03-08T18:43:46.024 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-packaging. 2022-03-08T18:43:46.033 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../86-python3-packaging_20.3-1_all.deb ... 2022-03-08T18:43:46.083 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-packaging (20.3-1) ... 2022-03-08T18:43:46.262 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-repoze.lru (0.7-2) ... 2022-03-08T18:43:46.520 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-pastescript. 2022-03-08T18:43:46.536 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../87-python3-pastescript_2.0.2-4_all.deb ... 2022-03-08T18:43:46.572 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up liboath0 (2.6.1-1.3) ... 2022-03-08T18:43:46.586 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2022-03-08T18:43:46.698 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2022-03-08T18:43:46.974 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-pluggy. 2022-03-08T18:43:46.984 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../88-python3-pluggy_0.13.0-2_all.deb ... 2022-03-08T18:43:47.024 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-joblib (0.14.0-3) ... 2022-03-08T18:43:47.032 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2022-03-08T18:43:47.182 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package ceph-volume. 2022-03-08T18:43:47.195 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../77-ceph-volume_17.0.0-10961-g6fe8e680-1focal_all.deb ... 2022-03-08T18:43:47.261 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking ceph-volume (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:47.394 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-psutil. 2022-03-08T18:43:47.403 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-jwt (1.7.1-2ubuntu2) ... 2022-03-08T18:43:47.408 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../89-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2022-03-08T18:43:47.461 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2022-03-08T18:43:47.747 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-cachetools (4.0.0-1) ... 2022-03-08T18:43:47.801 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package cephadm. 2022-03-08T18:43:47.815 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../78-cephadm_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:47.832 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-py. 2022-03-08T18:43:47.844 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../90-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2022-03-08T18:43:47.859 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking cephadm (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:47.881 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2022-03-08T18:43:48.040 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2022-03-08T18:43:48.143 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package libcephfs-dev. 2022-03-08T18:43:48.156 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../79-libcephfs-dev_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:48.201 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking libcephfs-dev (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:48.268 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-pyinotify. 2022-03-08T18:43:48.282 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../91-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2022-03-08T18:43:48.343 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2022-03-08T18:43:48.405 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2022-03-08T18:43:48.697 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package nvme-cli. 2022-03-08T18:43:48.714 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../80-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2022-03-08T18:43:48.756 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-ceph-argparse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:48.773 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-wcwidth. 2022-03-08T18:43:48.786 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2022-03-08T18:43:48.787 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../92-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2022-03-08T18:43:48.831 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2022-03-08T18:43:49.072 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2022-03-08T18:43:49.119 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-pytest. 2022-03-08T18:43:49.134 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../93-python3-pytest_4.6.9-1_all.deb ... 2022-03-08T18:43:49.177 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-pytest (4.6.9-1) ... 2022-03-08T18:43:49.217 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2022-03-08T18:43:49.465 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-atomicwrites. 2022-03-08T18:43:49.478 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../81-python3-atomicwrites_1.1.5-2build1_all.deb ... 2022-03-08T18:43:49.517 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2022-03-08T18:43:49.536 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2022-03-08T18:43:49.566 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package python3-simplejson. 2022-03-08T18:43:49.581 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../94-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2022-03-08T18:43:49.637 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2022-03-08T18:43:49.837 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2022-03-08T18:43:49.999 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-attr. 2022-03-08T18:43:50.015 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../82-python3-attr_19.3.0-2_all.deb ... 2022-03-08T18:43:50.057 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-attr (19.3.0-2) ... 2022-03-08T18:43:50.153 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package radosgw. 2022-03-08T18:43:50.167 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../95-radosgw_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:50.203 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking radosgw (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:50.238 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2022-03-08T18:43:50.441 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-webencodings. 2022-03-08T18:43:50.451 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../83-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2022-03-08T18:43:50.491 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2022-03-08T18:43:50.531 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2022-03-08T18:43:50.819 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-html5lib. 2022-03-08T18:43:50.833 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../84-python3-html5lib_1.0.1-2_all.deb ... 2022-03-08T18:43:50.892 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2022-03-08T18:43:50.911 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2022-03-08T18:43:51.244 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-tempita (0.5.2-6) ... 2022-03-08T18:43:51.478 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-lxml:amd64. 2022-03-08T18:43:51.492 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../85-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2022-03-08T18:43:51.545 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-packaging (20.3-1) ... 2022-03-08T18:43:51.547 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2022-03-08T18:43:51.667 INFO:teuthology.orchestra.run.smithi066.stdout:Selecting previously unselected package rbd-fuse. 2022-03-08T18:43:51.681 INFO:teuthology.orchestra.run.smithi066.stdout:Preparing to unpack .../96-rbd-fuse_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:51.734 INFO:teuthology.orchestra.run.smithi066.stdout:Unpacking rbd-fuse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:51.855 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2022-03-08T18:43:52.048 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2022-03-08T18:43:52.105 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-attr (19.3.0-2) ... 2022-03-08T18:43:52.152 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-packaging. 2022-03-08T18:43:52.169 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../86-python3-packaging_20.3-1_all.deb ... 2022-03-08T18:43:52.257 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-packaging (20.3-1) ... 2022-03-08T18:43:52.409 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2022-03-08T18:43:52.500 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2022-03-08T18:43:52.627 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-pastescript. 2022-03-08T18:43:52.634 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2022-03-08T18:43:52.640 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../87-python3-pastescript_2.0.2-4_all.deb ... 2022-03-08T18:43:52.693 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2022-03-08T18:43:52.802 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2) ... 2022-03-08T18:43:52.852 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up cephadm (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:53.123 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-pluggy. 2022-03-08T18:43:53.134 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../88-python3-pluggy_0.13.0-2_all.deb ... 2022-03-08T18:43:53.179 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2022-03-08T18:43:53.186 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2022-03-08T18:43:53.549 INFO:teuthology.orchestra.run.smithi066.stdout:Adding system user cephadm....done 2022-03-08T18:43:53.590 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-ceph-common (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:53.649 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-psutil. 2022-03-08T18:43:53.662 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../89-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2022-03-08T18:43:53.729 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-waitress (1.4.1-1) ... 2022-03-08T18:43:53.768 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2022-03-08T18:43:53.900 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-html5lib (1.0.1-2) ... 2022-03-08T18:43:54.090 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-repoze.lru (0.7-2) ... 2022-03-08T18:43:54.132 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-py. 2022-03-08T18:43:54.145 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../90-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2022-03-08T18:43:54.197 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2022-03-08T18:43:54.277 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2022-03-08T18:43:54.450 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up liboath0 (2.6.1-1.3) ... 2022-03-08T18:43:54.487 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-pluggy (0.13.0-2) ... 2022-03-08T18:43:54.619 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2022-03-08T18:43:54.712 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-pyinotify. 2022-03-08T18:43:54.724 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../91-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2022-03-08T18:43:54.761 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2022-03-08T18:43:54.797 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2022-03-08T18:43:54.970 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-joblib (0.14.0-3) ... 2022-03-08T18:43:55.082 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-wcwidth. 2022-03-08T18:43:55.095 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../92-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2022-03-08T18:43:55.131 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2022-03-08T18:43:55.449 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-jwt (1.7.1-2ubuntu2) ... 2022-03-08T18:43:55.474 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libxslt1.1:amd64 (1.1.34-4) ... 2022-03-08T18:43:55.516 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-pytest. 2022-03-08T18:43:55.525 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../93-python3-pytest_4.6.9-1_all.deb ... 2022-03-08T18:43:55.591 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-pytest (4.6.9-1) ... 2022-03-08T18:43:55.617 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2022-03-08T18:43:55.734 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-cachetools (4.0.0-1) ... 2022-03-08T18:43:55.995 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2022-03-08T18:43:56.104 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package python3-simplejson. 2022-03-08T18:43:56.119 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../94-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2022-03-08T18:43:56.152 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2022-03-08T18:43:56.153 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2022-03-08T18:43:56.271 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-logutils (0.3.3-7) ... 2022-03-08T18:43:56.446 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2022-03-08T18:43:56.523 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package radosgw. 2022-03-08T18:43:56.536 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../95-radosgw_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:56.589 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2022-03-08T18:43:56.609 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking radosgw (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:56.738 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-ceph-argparse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:56.757 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2022-03-08T18:43:57.051 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-prettytable (0.7.2-5) ... 2022-03-08T18:43:57.073 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2022-03-08T18:43:57.239 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2022-03-08T18:43:57.369 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2022-03-08T18:43:57.543 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2022-03-08T18:43:57.678 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2022-03-08T18:43:57.768 INFO:teuthology.orchestra.run.smithi137.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2022-03-08T18:43:57.853 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2022-03-08T18:43:58.005 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2022-03-08T18:43:58.157 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2) ... 2022-03-08T18:43:58.243 INFO:teuthology.orchestra.run.smithi002.stdout:Selecting previously unselected package rbd-fuse. 2022-03-08T18:43:58.258 INFO:teuthology.orchestra.run.smithi002.stdout:Preparing to unpack .../96-rbd-fuse_17.0.0-10961-g6fe8e680-1focal_amd64.deb ... 2022-03-08T18:43:58.313 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2022-03-08T18:43:58.332 INFO:teuthology.orchestra.run.smithi002.stdout:Unpacking rbd-fuse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:58.533 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2022-03-08T18:43:58.601 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2022-03-08T18:43:58.626 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-attr (19.3.0-2) ... 2022-03-08T18:43:58.894 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2022-03-08T18:43:58.960 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2022-03-08T18:43:59.003 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2022-03-08T18:43:59.020 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2022-03-08T18:43:59.129 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2022-03-08T18:43:59.146 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-webob (1:1.8.5-2) ... 2022-03-08T18:43:59.278 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-tempita (0.5.2-6) ... 2022-03-08T18:43:59.414 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up cephadm (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:59.475 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2022-03-08T18:43:59.631 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-packaging (20.3-1) ... 2022-03-08T18:43:59.848 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up librados2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:43:59.973 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2022-03-08T18:43:59.974 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2022-03-08T18:44:00.127 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2022-03-08T18:44:00.257 INFO:teuthology.orchestra.run.smithi002.stdout:Adding system user cephadm....done 2022-03-08T18:44:00.419 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-waitress (1.4.1-1) ... 2022-03-08T18:44:00.769 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-repoze.lru (0.7-2) ... 2022-03-08T18:44:01.027 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2022-03-08T18:44:01.090 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up liboath0 (2.6.1-1.3) ... 2022-03-08T18:44:01.221 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2022-03-08T18:44:01.463 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2) ... 2022-03-08T18:44:01.553 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up librgw2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:01.625 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-joblib (0.14.0-3) ... 2022-03-08T18:44:01.681 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libsqlite3-mod-ceph (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:01.818 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2022-03-08T18:44:01.840 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2022-03-08T18:44:01.958 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-pytest (4.6.9-1) ... 2022-03-08T18:44:02.117 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-jwt (1.7.1-2ubuntu2) ... 2022-03-08T18:44:02.219 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-ceph-common (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:02.408 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2022-03-08T18:44:02.562 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-html5lib (1.0.1-2) ... 2022-03-08T18:44:02.590 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-cachetools (4.0.0-1) ... 2022-03-08T18:44:02.719 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2022-03-08T18:44:02.928 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2022-03-08T18:44:02.938 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2022-03-08T18:44:03.014 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-bs4 (4.8.2-1) ... 2022-03-08T18:44:03.065 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-pluggy (0.13.0-2) ... 2022-03-08T18:44:03.230 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2022-03-08T18:44:03.333 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libcephfs2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:03.383 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2022-03-08T18:44:03.461 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2022-03-08T18:44:03.509 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libxslt1.1:amd64 (1.1.34-4) ... 2022-03-08T18:44:03.572 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-ceph-argparse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:03.635 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2022-03-08T18:44:03.785 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libradosstriper1 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:03.905 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2022-03-08T18:44:03.913 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2022-03-08T18:44:03.951 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2022-03-08T18:44:04.081 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2022-03-08T18:44:04.241 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2022-03-08T18:44:04.267 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-logutils (0.3.3-7) ... 2022-03-08T18:44:04.474 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2022-03-08T18:44:04.637 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2022-03-08T18:44:04.809 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2022-03-08T18:44:04.810 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2022-03-08T18:44:05.140 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-prettytable (0.7.2-5) ... 2022-03-08T18:44:05.177 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2022-03-08T18:44:05.418 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up librbd1 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:05.445 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2022-03-08T18:44:05.476 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2022-03-08T18:44:05.557 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2022-03-08T18:44:05.693 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2022-03-08T18:44:05.761 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2022-03-08T18:44:05.862 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2022-03-08T18:44:05.920 INFO:teuthology.orchestra.run.smithi066.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2022-03-08T18:44:05.921 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2022-03-08T18:44:06.197 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2022-03-08T18:44:06.228 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2022-03-08T18:44:06.296 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-tempita (0.5.2-6) ... 2022-03-08T18:44:06.352 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2) ... 2022-03-08T18:44:06.364 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-rsa (4.0-3) ... 2022-03-08T18:44:06.606 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-packaging (20.3-1) ... 2022-03-08T18:44:06.675 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-webtest (2.0.32-2) ... 2022-03-08T18:44:06.725 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2022-03-08T18:44:06.957 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2022-03-08T18:44:06.980 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph-mgr-modules-core (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:07.088 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2022-03-08T18:44:07.105 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2022-03-08T18:44:07.150 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph-fuse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:07.284 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2022-03-08T18:44:07.393 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2022-03-08T18:44:07.393 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2022-03-08T18:44:07.446 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-webob (1:1.8.5-2) ... 2022-03-08T18:44:07.791 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2022-03-08T18:44:07.798 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2022-03-08T18:44:08.084 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up librados2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:08.237 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2022-03-08T18:44:08.280 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up libcephfs-dev (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:08.341 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2) ... 2022-03-08T18:44:08.508 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2022-03-08T18:44:08.768 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2022-03-08T18:44:08.805 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-pastescript (2.0.2-4) ... 2022-03-08T18:44:09.133 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2022-03-08T18:44:09.152 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-ceph-common (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:09.471 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-html5lib (1.0.1-2) ... 2022-03-08T18:44:09.494 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-rados (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:09.724 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-rbd (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:09.866 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up librgw2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:09.889 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2022-03-08T18:44:09.927 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up rbd-fuse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:10.052 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libsqlite3-mod-ceph (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:10.071 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-routes (2.4.1-2) ... 2022-03-08T18:44:10.107 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-pluggy (0.13.0-2) ... 2022-03-08T18:44:10.180 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2022-03-08T18:44:10.308 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-pytest (4.6.9-1) ... 2022-03-08T18:44:10.349 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-google-auth (1.5.1-2) ... 2022-03-08T18:44:10.433 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2022-03-08T18:44:10.558 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libxslt1.1:amd64 (1.1.34-4) ... 2022-03-08T18:44:10.652 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-rgw (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:10.685 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2022-03-08T18:44:10.698 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2022-03-08T18:44:10.868 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-cephfs (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:11.042 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2022-03-08T18:44:11.052 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2022-03-08T18:44:11.085 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2022-03-08T18:44:11.387 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-logutils (0.3.3-7) ... 2022-03-08T18:44:11.406 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-bs4 (4.8.2-1) ... 2022-03-08T18:44:11.694 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2022-03-08T18:44:11.751 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libcephfs2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:11.828 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2022-03-08T18:44:11.879 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2022-03-08T18:44:12.137 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph-common (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:12.228 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libradosstriper1 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:12.241 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-prettytable (0.7.2-5) ... 2022-03-08T18:44:12.373 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2022-03-08T18:44:12.589 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2022-03-08T18:44:12.727 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2022-03-08T18:44:12.791 INFO:teuthology.orchestra.run.smithi137.stdout:Adding group ceph....done 2022-03-08T18:44:12.940 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2022-03-08T18:44:13.054 INFO:teuthology.orchestra.run.smithi002.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2022-03-08T18:44:13.137 INFO:teuthology.orchestra.run.smithi137.stdout:Adding system user ceph....done 2022-03-08T18:44:13.349 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2022-03-08T18:44:13.419 INFO:teuthology.orchestra.run.smithi137.stdout:Setting system user ceph properties....done 2022-03-08T18:44:13.432 INFO:teuthology.orchestra.run.smithi137.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2022-03-08T18:44:13.574 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2022-03-08T18:44:13.584 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2) ... 2022-03-08T18:44:13.886 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2022-03-08T18:44:13.907 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up librbd1 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:14.035 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2022-03-08T18:44:14.046 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2022-03-08T18:44:14.221 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2022-03-08T18:44:14.311 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2022-03-08T18:44:14.358 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2022-03-08T18:44:14.529 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2022-03-08T18:44:14.737 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-webob (1:1.8.5-2) ... 2022-03-08T18:44:14.737 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2022-03-08T18:44:14.884 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph-test (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:14.898 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-rsa (4.0-3) ... 2022-03-08T18:44:15.039 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up radosgw (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:15.097 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2022-03-08T18:44:15.259 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-webtest (2.0.32-2) ... 2022-03-08T18:44:15.299 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2022-03-08T18:44:15.299 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2022-03-08T18:44:15.399 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up librados2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:15.583 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph-mgr-modules-core (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:15.632 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2022-03-08T18:44:15.736 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph-fuse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:15.973 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2022-03-08T18:44:15.973 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2022-03-08T18:44:16.644 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph-base (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:16.798 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up libcephfs-dev (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:16.929 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2022-03-08T18:44:17.001 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2022-03-08T18:44:17.259 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-pastescript (2.0.2-4) ... 2022-03-08T18:44:17.324 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up librgw2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:17.449 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libsqlite3-mod-ceph (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:17.598 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2022-03-08T18:44:17.617 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2022-03-08T18:44:17.750 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-pytest (4.6.9-1) ... 2022-03-08T18:44:17.839 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph-mds (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:18.049 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-rados (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:18.079 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2022-03-08T18:44:18.080 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2022-03-08T18:44:18.144 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2022-03-08T18:44:18.229 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-rbd (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:18.434 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up rbd-fuse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:18.453 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2022-03-08T18:44:18.562 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-routes (2.4.1-2) ... 2022-03-08T18:44:18.763 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-bs4 (4.8.2-1) ... 2022-03-08T18:44:18.862 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-google-auth (1.5.1-2) ... 2022-03-08T18:44:18.946 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph-mgr (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:19.147 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libcephfs2 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:19.187 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2022-03-08T18:44:19.187 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2022-03-08T18:44:19.214 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-rgw (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:19.315 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2022-03-08T18:44:19.361 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-cephfs (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:19.581 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2022-03-08T18:44:19.678 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libradosstriper1 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:19.835 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2022-03-08T18:44:20.060 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph-osd (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:20.204 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2022-03-08T18:44:20.775 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph-common (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:20.788 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2022-03-08T18:44:20.789 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2022-03-08T18:44:21.450 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up librbd1 (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:21.507 INFO:teuthology.orchestra.run.smithi066.stdout:Adding group ceph....done 2022-03-08T18:44:21.635 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph-mgr-k8sevents (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:21.656 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2022-03-08T18:44:21.832 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph-mgr-diskprediction-local (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:21.931 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2022-03-08T18:44:22.030 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph-mon (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:22.069 INFO:teuthology.orchestra.run.smithi066.stdout:Adding system user ceph....done 2022-03-08T18:44:22.171 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2022-03-08T18:44:22.244 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2022-03-08T18:44:22.244 INFO:teuthology.orchestra.run.smithi137.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2022-03-08T18:44:22.264 INFO:teuthology.orchestra.run.smithi066.stdout:Setting system user ceph properties....done 2022-03-08T18:44:22.285 INFO:teuthology.orchestra.run.smithi066.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2022-03-08T18:44:22.424 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2022-03-08T18:44:22.501 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2022-03-08T18:44:22.632 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-rsa (4.0-3) ... 2022-03-08T18:44:22.958 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2022-03-08T18:44:23.017 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-webtest (2.0.32-2) ... 2022-03-08T18:44:23.023 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:23.188 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph-mgr-dashboard (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:23.335 INFO:teuthology.orchestra.run.smithi137.stdout:Setting up ceph-volume (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:23.365 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph-mgr-modules-core (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:23.474 INFO:teuthology.orchestra.run.smithi137.stdout:Processing triggers for systemd (245.4-4ubuntu3.15) ... 2022-03-08T18:44:23.495 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph-fuse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:23.772 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2022-03-08T18:44:23.772 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2022-03-08T18:44:23.793 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph-test (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:23.965 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up radosgw (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:23.987 INFO:teuthology.orchestra.run.smithi137.stdout:Processing triggers for man-db (2.9.1-1) ... 2022-03-08T18:44:24.255 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2022-03-08T18:44:24.256 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2022-03-08T18:44:24.617 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up libcephfs-dev (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:24.763 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2022-03-08T18:44:25.070 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-pastescript (2.0.2-4) ... 2022-03-08T18:44:25.407 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2022-03-08T18:44:25.496 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph-base (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:25.775 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-rados (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:25.872 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2022-03-08T18:44:25.960 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-rbd (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:26.165 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up rbd-fuse (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:26.320 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-routes (2.4.1-2) ... 2022-03-08T18:44:26.653 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph-mds (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:26.656 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-google-auth (1.5.1-2) ... 2022-03-08T18:44:26.885 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2022-03-08T18:44:26.885 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2022-03-08T18:44:26.988 INFO:teuthology.orchestra.run.smithi137.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.2) ... 2022-03-08T18:44:27.065 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-rgw (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:27.228 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-cephfs (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:27.391 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2022-03-08T18:44:27.471 INFO:teuthology.orchestra.run.smithi137.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2022-03-08T18:44:27.475 DEBUG:teuthology.parallel:result is None 2022-03-08T18:44:27.713 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph-mgr (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:27.942 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2022-03-08T18:44:27.942 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2022-03-08T18:44:28.532 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph-common (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:28.729 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph-osd (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:29.139 INFO:teuthology.orchestra.run.smithi002.stdout:Adding group ceph....done 2022-03-08T18:44:29.411 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2022-03-08T18:44:29.411 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2022-03-08T18:44:29.735 INFO:teuthology.orchestra.run.smithi002.stdout:Adding system user ceph....done 2022-03-08T18:44:29.897 INFO:teuthology.orchestra.run.smithi002.stdout:Setting system user ceph properties....done 2022-03-08T18:44:29.914 INFO:teuthology.orchestra.run.smithi002.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2022-03-08T18:44:30.072 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2022-03-08T18:44:30.258 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph-mgr-k8sevents (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:30.386 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph-mgr-diskprediction-local (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:30.570 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph-mon (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:30.584 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2022-03-08T18:44:30.820 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2022-03-08T18:44:30.821 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2022-03-08T18:44:31.437 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph-test (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:31.563 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up radosgw (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:31.921 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2022-03-08T18:44:31.921 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2022-03-08T18:44:32.083 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:32.258 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph-mgr-dashboard (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:32.473 INFO:teuthology.orchestra.run.smithi066.stdout:Setting up ceph-volume (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:32.613 INFO:teuthology.orchestra.run.smithi066.stdout:Processing triggers for systemd (245.4-4ubuntu3.15) ... 2022-03-08T18:44:33.098 INFO:teuthology.orchestra.run.smithi066.stdout:Processing triggers for man-db (2.9.1-1) ... 2022-03-08T18:44:33.126 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph-base (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:33.638 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2022-03-08T18:44:34.534 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph-mds (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:34.810 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2022-03-08T18:44:34.811 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2022-03-08T18:44:35.662 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph-mgr (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:35.897 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2022-03-08T18:44:35.897 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2022-03-08T18:44:35.976 INFO:teuthology.orchestra.run.smithi066.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.2) ... 2022-03-08T18:44:36.518 INFO:teuthology.orchestra.run.smithi066.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2022-03-08T18:44:36.521 DEBUG:teuthology.parallel:result is None 2022-03-08T18:44:36.718 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph-osd (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:37.469 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2022-03-08T18:44:37.469 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2022-03-08T18:44:38.306 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph-mgr-k8sevents (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:38.553 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph-mgr-diskprediction-local (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:39.773 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph-mon (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:40.081 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2022-03-08T18:44:40.082 INFO:teuthology.orchestra.run.smithi002.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2022-03-08T18:44:40.860 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:41.028 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph-mgr-dashboard (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:41.305 INFO:teuthology.orchestra.run.smithi002.stdout:Setting up ceph-volume (17.0.0-10961-g6fe8e680-1focal) ... 2022-03-08T18:44:41.441 INFO:teuthology.orchestra.run.smithi002.stdout:Processing triggers for systemd (245.4-4ubuntu3.15) ... 2022-03-08T18:44:42.011 INFO:teuthology.orchestra.run.smithi002.stdout:Processing triggers for man-db (2.9.1-1) ... 2022-03-08T18:44:44.843 INFO:teuthology.orchestra.run.smithi002.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.2) ... 2022-03-08T18:44:45.409 INFO:teuthology.orchestra.run.smithi002.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2022-03-08T18:44:45.411 DEBUG:teuthology.parallel:result is None 2022-03-08T18:44:45.412 DEBUG:teuthology.orchestra.run.smithi002:> sudo lsb_release -is 2022-03-08T18:44:45.464 INFO:teuthology.orchestra.run.smithi002.stdout:Ubuntu 2022-03-08T18:44:45.469 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-03-08T18:44:45.470 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&sha1=6fe8e680a334072b1ba19b2de006862f98aedbae 2022-03-08T18:44:45.628 DEBUG:teuthology.orchestra.run.smithi002:> dpkg-query -W -f '${Version}' ceph 2022-03-08T18:44:45.651 INFO:teuthology.orchestra.run.smithi002.stdout:17.0.0-10961-g6fe8e680-1focal 2022-03-08T18:44:45.652 INFO:teuthology.packaging:The installed version of ceph is 17.0.0-10961-g6fe8e680-1focal 2022-03-08T18:44:45.653 INFO:teuthology.task.install:The correct ceph version 17.0.0-10961-g6fe8e680-1focal is installed. 2022-03-08T18:44:45.653 DEBUG:teuthology.orchestra.run.smithi066:> sudo lsb_release -is 2022-03-08T18:44:45.705 INFO:teuthology.orchestra.run.smithi066.stdout:Ubuntu 2022-03-08T18:44:45.711 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-03-08T18:44:45.712 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&sha1=6fe8e680a334072b1ba19b2de006862f98aedbae 2022-03-08T18:44:45.867 DEBUG:teuthology.orchestra.run.smithi066:> dpkg-query -W -f '${Version}' ceph 2022-03-08T18:44:45.887 INFO:teuthology.orchestra.run.smithi066.stdout:17.0.0-10961-g6fe8e680-1focal 2022-03-08T18:44:45.888 INFO:teuthology.packaging:The installed version of ceph is 17.0.0-10961-g6fe8e680-1focal 2022-03-08T18:44:45.888 INFO:teuthology.task.install:The correct ceph version 17.0.0-10961-g6fe8e680-1focal is installed. 2022-03-08T18:44:45.889 DEBUG:teuthology.orchestra.run.smithi137:> sudo lsb_release -is 2022-03-08T18:44:45.940 INFO:teuthology.orchestra.run.smithi137.stdout:Ubuntu 2022-03-08T18:44:45.945 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-03-08T18:44:45.946 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&sha1=6fe8e680a334072b1ba19b2de006862f98aedbae 2022-03-08T18:44:46.097 DEBUG:teuthology.orchestra.run.smithi137:> dpkg-query -W -f '${Version}' ceph 2022-03-08T18:44:46.120 INFO:teuthology.orchestra.run.smithi137.stdout:17.0.0-10961-g6fe8e680-1focal 2022-03-08T18:44:46.121 INFO:teuthology.packaging:The installed version of ceph is 17.0.0-10961-g6fe8e680-1focal 2022-03-08T18:44:46.121 INFO:teuthology.task.install:The correct ceph version 17.0.0-10961-g6fe8e680-1focal is installed. 2022-03-08T18:44:46.122 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2022-03-08T18:44:46.123 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:44:46.123 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2022-03-08T18:44:46.139 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:44:46.139 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2022-03-08T18:44:46.156 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:44:46.157 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2022-03-08T18:44:46.178 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2022-03-08T18:44:46.179 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:44:46.179 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd of=/usr/bin/daemon-helper 2022-03-08T18:44:46.195 DEBUG:teuthology.orchestra.run.smithi002:> sudo chmod a=rx -- /usr/bin/daemon-helper 2022-03-08T18:44:46.206 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:44:46.206 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/usr/bin/daemon-helper 2022-03-08T18:44:46.226 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod a=rx -- /usr/bin/daemon-helper 2022-03-08T18:44:46.284 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:44:46.285 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd of=/usr/bin/daemon-helper 2022-03-08T18:44:46.300 DEBUG:teuthology.orchestra.run.smithi137:> sudo chmod a=rx -- /usr/bin/daemon-helper 2022-03-08T18:44:46.312 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2022-03-08T18:44:46.312 DEBUG:teuthology.orchestra.run.smithi002:> set -ex 2022-03-08T18:44:46.313 DEBUG:teuthology.orchestra.run.smithi002:> sudo dd of=/usr/bin/adjust-ulimits 2022-03-08T18:44:46.332 DEBUG:teuthology.orchestra.run.smithi002:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2022-03-08T18:44:46.393 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2022-03-08T18:44:46.394 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/usr/bin/adjust-ulimits 2022-03-08T18:44:46.408 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2022-03-08T18:44:46.473 DEBUG:teuthology.orchestra.run.smithi137:> set -ex 2022-03-08T18:44:46.473 DEBUG:teuthology.orchestra.run.smithi137:> sudo dd of=/usr/bin/adjust-ulimits 2022-03-08T18:44:46.500 DEBUG:teuthology.orchestra.run.smithi137:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2022-03-08T18:44:46.515 INFO:teuthology.run_tasks:Running task radosbench... 2022-03-08T18:44:46.540 INFO:tasks.radosbench:Beginning radosbench... 2022-03-08T18:44:46.540 INFO:tasks.rook.ceph_manager.ceph:creating pool_name unique_pool_0 2022-03-08T18:44:46.541 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd pool create unique_pool_0 16 2022-03-08T18:44:48.232 INFO:teuthology.orchestra.run.smithi002.stderr:pool 'unique_pool_0' created 2022-03-08T18:44:48.250 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2022-03-08T18:44:49.470 INFO:teuthology.orchestra.run.smithi002.stderr:enabled application 'rados' on pool 'unique_pool_0' 2022-03-08T18:44:50.494 DEBUG:teuthology.orchestra.run.smithi002:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.a -b 65536 --object-size 65536 -p unique_pool_0 bench 360 write' 2022-03-08T18:44:50.497 INFO:teuthology.run_tasks:Running task rook.shell... 2022-03-08T18:44:50.515 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch host label add '`hostname`' 'foo 2022-03-08T18:44:50.515 DEBUG:teuthology.orchestra.run.smithi002:> ceph' orch host ls '|' grep 'foo 2022-03-08T18:44:50.515 DEBUG:teuthology.orchestra.run.smithi002:> ceph' orch host label rm '`hostname`' 'foo 2022-03-08T18:44:50.516 DEBUG:teuthology.orchestra.run.smithi002:> ceph' orch host ls '|' grep -v 'foo 2022-03-08T18:44:50.516 DEBUG:teuthology.orchestra.run.smithi002:> ' 2022-03-08T18:44:50.593 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:hints = 1 2022-03-08T18:44:50.593 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 360 seconds or 0 objects 2022-03-08T18:44:50.594 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Object prefix: benchmark_data_smithi002_47145 2022-03-08T18:44:50.594 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:44:50.595 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 0 0 0 0 0 0 - 0 2022-03-08T18:44:50.861 INFO:teuthology.orchestra.run.smithi002.stdout:ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable) 2022-03-08T18:44:50.888 INFO:teuthology.run_tasks:Running task exec... 2022-03-08T18:44:50.901 INFO:teuthology.task.exec:Executing custom commands... 2022-03-08T18:44:50.901 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi002.front.sepia.ceph.com 2022-03-08T18:44:50.902 DEBUG:teuthology.orchestra.run.smithi002:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'set -ex 2022-03-08T18:44:50.902 DEBUG:teuthology.orchestra.run.smithi002:> toolbox() { 2022-03-08T18:44:50.902 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- "$@" 2022-03-08T18:44:50.903 DEBUG:teuthology.orchestra.run.smithi002:> } 2022-03-08T18:44:50.903 DEBUG:teuthology.orchestra.run.smithi002:> orig_num_osd=`toolbox ceph osd stat | cut -f3 -d " "` 2022-03-08T18:44:50.903 DEBUG:teuthology.orchestra.run.smithi002:> toolbox ceph orch osd rm 0 --force 2022-03-08T18:44:50.904 DEBUG:teuthology.orchestra.run.smithi002:> removed_pv="" 2022-03-08T18:44:50.904 DEBUG:teuthology.orchestra.run.smithi002:> while [ "$removed_pv" = "" ] 2022-03-08T18:44:50.904 DEBUG:teuthology.orchestra.run.smithi002:> do 2022-03-08T18:44:50.904 DEBUG:teuthology.orchestra.run.smithi002:> removed_pv=`kubectl get pv | grep Released | cut -f1 -d " "` 2022-03-08T18:44:50.905 DEBUG:teuthology.orchestra.run.smithi002:> sleep 3s 2022-03-08T18:44:50.905 DEBUG:teuthology.orchestra.run.smithi002:> done 2022-03-08T18:44:50.905 DEBUG:teuthology.orchestra.run.smithi002:> target_path=`kubectl get pv $removed_pv -o jsonpath='"'"'{.spec.local.path}'"'"'` 2022-03-08T18:44:50.906 DEBUG:teuthology.orchestra.run.smithi002:> host=`echo $removed_pv | cut -f1 -d "-"` 2022-03-08T18:44:50.906 DEBUG:teuthology.orchestra.run.smithi002:> toolbox ceph orch device zap $host $target_path --force 2022-03-08T18:44:50.906 DEBUG:teuthology.orchestra.run.smithi002:> zap_completion="0" 2022-03-08T18:44:50.907 DEBUG:teuthology.orchestra.run.smithi002:> while [ "$zap_completion" = "0" ] 2022-03-08T18:44:50.907 DEBUG:teuthology.orchestra.run.smithi002:> do 2022-03-08T18:44:50.907 DEBUG:teuthology.orchestra.run.smithi002:> zap_completion=`kubectl get job -n rook-ceph rook-ceph-device-zap -o jsonpath='"'"'{.status.succeeded.path}'"'"'` 2022-03-08T18:44:50.907 DEBUG:teuthology.orchestra.run.smithi002:> sleep 3s 2022-03-08T18:44:50.908 DEBUG:teuthology.orchestra.run.smithi002:> done 2022-03-08T18:44:50.908 DEBUG:teuthology.orchestra.run.smithi002:> kubectl patch pv $removed_pv -p '"'"'{"spec":{"claimRef": null}}'"'"' 2022-03-08T18:44:50.908 DEBUG:teuthology.orchestra.run.smithi002:> toolbox ceph orch apply osd --all-available-devices 2022-03-08T18:44:50.909 DEBUG:teuthology.orchestra.run.smithi002:> kubectl delete job rook-ceph-device-zap -n rook-ceph 2022-03-08T18:44:50.909 DEBUG:teuthology.orchestra.run.smithi002:> num_osd="0" 2022-03-08T18:44:50.909 DEBUG:teuthology.orchestra.run.smithi002:> while [ "$num_osd" != "$orig_num_osd" ] 2022-03-08T18:44:50.909 DEBUG:teuthology.orchestra.run.smithi002:> do 2022-03-08T18:44:50.910 DEBUG:teuthology.orchestra.run.smithi002:> echo "waiting for osd to come back up" 2022-03-08T18:44:50.910 DEBUG:teuthology.orchestra.run.smithi002:> num_osd=`toolbox ceph osd stat | cut -f3 -d " "` 2022-03-08T18:44:50.910 DEBUG:teuthology.orchestra.run.smithi002:> sleep 30s 2022-03-08T18:44:50.911 DEBUG:teuthology.orchestra.run.smithi002:> done 2022-03-08T18:44:50.911 DEBUG:teuthology.orchestra.run.smithi002:> ' 2022-03-08T18:44:50.924 INFO:teuthology.orchestra.run.smithi002.stderr:++ toolbox ceph osd stat 2022-03-08T18:44:50.924 INFO:teuthology.orchestra.run.smithi002.stderr:++ kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd stat 2022-03-08T18:44:50.924 INFO:teuthology.orchestra.run.smithi002.stderr:++ cut -f3 -d ' ' 2022-03-08T18:44:51.594 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 1 16 2799 2783 173.928 173.938 0.00547896 0.00573468 2022-03-08T18:44:52.595 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 2 16 5559 5543 173.206 172.5 0.00355396 0.00575999 2022-03-08T18:44:53.481 INFO:teuthology.orchestra.run.smithi002.stderr:Unable to use a TTY - input is not a terminal or the right kind of file 2022-03-08T18:44:53.594 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 3 16 8072 8056 167.816 157.062 0.010264 0.00594751 2022-03-08T18:44:54.127 INFO:teuthology.orchestra.run.smithi002.stderr:+ orig_num_osd=12 2022-03-08T18:44:54.127 INFO:teuthology.orchestra.run.smithi002.stderr:+ toolbox ceph orch osd rm 0 --force 2022-03-08T18:44:54.128 INFO:teuthology.orchestra.run.smithi002.stderr:+ kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph orch osd rm 0 --force 2022-03-08T18:44:54.228 INFO:teuthology.orchestra.run.smithi002.stderr:Unable to use a TTY - input is not a terminal or the right kind of file 2022-03-08T18:44:54.594 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 4 16 10449 10433 162.999 148.562 0.00298548 0.00612608 2022-03-08T18:44:55.114 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:44:55.102743 I | ceph-cluster-controller: CR has changed for "rook-ceph". diff=  v1.ClusterSpec{ 2022-03-08T18:44:55.114 INFO:tasks.rook.operator.smithi002.stdout:   CephVersion: {Image: "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", AllowUnsupported: true}, 2022-03-08T18:44:55.114 INFO:tasks.rook.operator.smithi002.stdout:   Storage: v1.StorageScopeSpec{ 2022-03-08T18:44:55.115 INFO:tasks.rook.operator.smithi002.stdout:   ... // 3 identical fields 2022-03-08T18:44:55.115 INFO:tasks.rook.operator.smithi002.stdout:   Config: nil, 2022-03-08T18:44:55.116 INFO:tasks.rook.operator.smithi002.stdout:   Selection: {}, 2022-03-08T18:44:55.116 INFO:tasks.rook.operator.smithi002.stdout:   StorageClassDeviceSets: []v1.StorageClassDeviceSet{ 2022-03-08T18:44:55.116 INFO:tasks.rook.operator.smithi002.stdout:   ... // 2 identical elements 2022-03-08T18:44:55.116 INFO:tasks.rook.operator.smithi002.stdout:   {Name: "smithi002-lv-3", Count: 1, VolumeClaimTemplates: {{ObjectMeta: {Name: "data"}, Spec: {AccessModes: {"ReadWriteOnce"}, Resources: {Requests: {s"storage": {i: {...}, s: "1", Format: "DecimalSI"}}}, VolumeName: "smithi002-lv-3", StorageClassName: &"scratch", ...}}}}, 2022-03-08T18:44:55.117 INFO:tasks.rook.operator.smithi002.stdout:   {Name: "smithi002-lv-4", Count: 1, VolumeClaimTemplates: {{ObjectMeta: {Name: "data"}, Spec: {AccessModes: {"ReadWriteOnce"}, Resources: {Requests: {s"storage": {i: {...}, s: "1", Format: "DecimalSI"}}}, VolumeName: "smithi002-lv-4", StorageClassName: &"scratch", ...}}}}, 2022-03-08T18:44:55.117 INFO:tasks.rook.operator.smithi002.stdout:-  { 2022-03-08T18:44:55.117 INFO:tasks.rook.operator.smithi002.stdout:-  Name: "smithi066-lv-1", 2022-03-08T18:44:55.118 INFO:tasks.rook.operator.smithi002.stdout:-  Count: 1, 2022-03-08T18:44:55.118 INFO:tasks.rook.operator.smithi002.stdout:-  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{ObjectMeta: v1.ObjectMeta{...}, Spec: v1.PersistentVolumeClaimSpec{...}}}, 2022-03-08T18:44:55.118 INFO:tasks.rook.operator.smithi002.stdout:-  }, 2022-03-08T18:44:55.118 INFO:tasks.rook.operator.smithi002.stdout:   {Name: "smithi066-lv-2", Count: 1, VolumeClaimTemplates: {{ObjectMeta: {Name: "data"}, Spec: {AccessModes: {"ReadWriteOnce"}, Resources: {Requests: {s"storage": {i: {...}, s: "1", Format: "DecimalSI"}}}, VolumeName: "smithi066-lv-2", StorageClassName: &"scratch", ...}}}}, 2022-03-08T18:44:55.119 INFO:tasks.rook.operator.smithi002.stdout:   {Name: "smithi066-lv-3", Count: 1, VolumeClaimTemplates: {{ObjectMeta: {Name: "data"}, Spec: {AccessModes: {"ReadWriteOnce"}, Resources: {Requests: {s"storage": {i: {...}, s: "1", Format: "DecimalSI"}}}, VolumeName: "smithi066-lv-3", StorageClassName: &"scratch", ...}}}}, 2022-03-08T18:44:55.119 INFO:tasks.rook.operator.smithi002.stdout:   ... // 5 identical elements 2022-03-08T18:44:55.119 INFO:tasks.rook.operator.smithi002.stdout:   }, 2022-03-08T18:44:55.119 INFO:tasks.rook.operator.smithi002.stdout:   }, 2022-03-08T18:44:55.120 INFO:tasks.rook.operator.smithi002.stdout:   Annotations: nil, 2022-03-08T18:44:55.120 INFO:tasks.rook.operator.smithi002.stdout:   Labels: nil, 2022-03-08T18:44:55.120 INFO:tasks.rook.operator.smithi002.stdout:   ... // 20 identical fields 2022-03-08T18:44:55.121 INFO:tasks.rook.operator.smithi002.stdout:  } 2022-03-08T18:44:55.121 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:44:55.102888 I | ceph-cluster-controller: reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:44:55.503 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:44:55.500474 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:44:55.594 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 5 16 12830 12814 160.159 148.812 0.00505331 0.00623526 2022-03-08T18:44:55.728 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:44:55.724704 I | ceph-cluster-controller: detecting the ceph image version for image quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae... 2022-03-08T18:44:56.595 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 6 16 13370 13354 139.09 33.75 0.00287978 0.00626998 2022-03-08T18:44:57.595 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 7 16 13370 13354 119.219 0 - 0.00626998 2022-03-08T18:44:58.595 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 8 16 13370 13354 104.316 0 - 0.00626998 2022-03-08T18:44:59.595 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 9 16 13370 13354 92.7247 0 - 0.00626998 2022-03-08T18:45:00.595 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 10 16 13370 13354 83.4522 0 - 0.00626998 2022-03-08T18:45:01.595 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 11 16 13370 13354 75.8655 0 - 0.00626998 2022-03-08T18:45:01.897 INFO:teuthology.orchestra.run.smithi002.stdout:Success 2022-03-08T18:45:01.951 INFO:teuthology.orchestra.run.smithi002.stderr:+ removed_pv= 2022-03-08T18:45:01.951 INFO:teuthology.orchestra.run.smithi002.stderr:+ '[' '' = '' ']' 2022-03-08T18:45:01.952 INFO:teuthology.orchestra.run.smithi002.stderr:++ kubectl get pv 2022-03-08T18:45:01.952 INFO:teuthology.orchestra.run.smithi002.stderr:++ grep Released 2022-03-08T18:45:01.952 INFO:teuthology.orchestra.run.smithi002.stderr:++ cut -f1 -d ' ' 2022-03-08T18:45:02.016 INFO:teuthology.orchestra.run.smithi002.stderr:+ removed_pv= 2022-03-08T18:45:02.017 INFO:teuthology.orchestra.run.smithi002.stderr:+ sleep 3s 2022-03-08T18:45:02.595 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 12 16 13370 13354 69.5435 0 - 0.00626998 2022-03-08T18:45:03.331 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:03.328552 I | ceph-spec: object "rook-ceph-osd-0" matched on delete, reconciling 2022-03-08T18:45:03.595 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 13 16 13429 13413 64.4776 0.526786 0.00265162 0.0151059 2022-03-08T18:45:04.595 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 14 16 13961 13945 62.2469 33.25 0.0100032 0.0160541 2022-03-08T18:45:05.018 INFO:teuthology.orchestra.run.smithi002.stderr:+ '[' '' = '' ']' 2022-03-08T18:45:05.019 INFO:teuthology.orchestra.run.smithi002.stderr:++ kubectl get pv 2022-03-08T18:45:05.019 INFO:teuthology.orchestra.run.smithi002.stderr:++ cut -f1 -d ' ' 2022-03-08T18:45:05.021 INFO:teuthology.orchestra.run.smithi002.stderr:++ grep Released 2022-03-08T18:45:05.141 INFO:teuthology.orchestra.run.smithi002.stderr:+ removed_pv=smithi066-lv-1 2022-03-08T18:45:05.141 INFO:teuthology.orchestra.run.smithi002.stderr:+ sleep 3s 2022-03-08T18:45:05.596 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 15 16 15684 15668 65.2754 107.688 0.0081898 0.0153114 2022-03-08T18:45:06.596 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 16 16 17625 17609 68.7769 121.312 0.0112753 0.0145319 2022-03-08T18:45:07.596 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 17 16 19221 19205 70.5983 99.75 0.00716693 0.0141572 2022-03-08T18:45:08.142 INFO:teuthology.orchestra.run.smithi002.stderr:+ '[' smithi066-lv-1 = '' ']' 2022-03-08T18:45:08.143 INFO:teuthology.orchestra.run.smithi002.stderr:++ kubectl get pv smithi066-lv-1 -o 'jsonpath={.spec.local.path}' 2022-03-08T18:45:08.249 INFO:teuthology.orchestra.run.smithi002.stderr:+ target_path=/dev/vg_nvme/lv_1 2022-03-08T18:45:08.252 INFO:teuthology.orchestra.run.smithi002.stderr:++ cut -f1 -d - 2022-03-08T18:45:08.252 INFO:teuthology.orchestra.run.smithi002.stderr:++ echo smithi066-lv-1 2022-03-08T18:45:08.252 INFO:teuthology.orchestra.run.smithi002.stderr:+ host=smithi066 2022-03-08T18:45:08.253 INFO:teuthology.orchestra.run.smithi002.stderr:+ toolbox ceph orch device zap smithi066 /dev/vg_nvme/lv_1 --force 2022-03-08T18:45:08.253 INFO:teuthology.orchestra.run.smithi002.stderr:+ kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph orch device zap smithi066 /dev/vg_nvme/lv_1 --force 2022-03-08T18:45:08.350 INFO:teuthology.orchestra.run.smithi002.stderr:Unable to use a TTY - input is not a terminal or the right kind of file 2022-03-08T18:45:08.596 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 18 16 20967 20951 72.7383 109.125 0.00371935 0.0137044 2022-03-08T18:45:09.129 INFO:teuthology.orchestra.run.smithi002.stdout:/dev/vg_nvme/lv_1 on smithi066 zapped 2022-03-08T18:45:09.150 INFO:teuthology.orchestra.run.smithi002.stderr:+ zap_completion=0 2022-03-08T18:45:09.150 INFO:teuthology.orchestra.run.smithi002.stderr:+ '[' 0 = 0 ']' 2022-03-08T18:45:09.151 INFO:teuthology.orchestra.run.smithi002.stderr:++ kubectl get job -n rook-ceph rook-ceph-device-zap -o 'jsonpath={.status.succeeded.path}' 2022-03-08T18:45:09.280 INFO:teuthology.orchestra.run.smithi002.stderr:+ zap_completion= 2022-03-08T18:45:09.281 INFO:teuthology.orchestra.run.smithi002.stderr:+ sleep 3s 2022-03-08T18:45:09.323 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:09.320950 I | ceph-cluster-controller: detected ceph image version: "17.0.0-10961 quincy" 2022-03-08T18:45:09.324 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:09.320982 I | ceph-cluster-controller: validating ceph version from provided image 2022-03-08T18:45:09.324 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:09.321000 W | ceph-cluster-controller: unsupported ceph version detected: "17.0.0-10961 quincy", pursuing 2022-03-08T18:45:09.354 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:09.351581 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:45:09.425 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:09.422967 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:45:09.425 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:09.423139 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:45:09.596 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 19 16 21601 21585 70.9953 39.625 0.00400767 0.0140592 2022-03-08T18:45:10.076 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:10.073205 I | ceph-cluster-controller: cluster "rook-ceph": version "17.0.0-10961 quincy" detected for image "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae" 2022-03-08T18:45:10.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:10.190943 I | op-mon: start running mons 2022-03-08T18:45:10.220 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:10.214356 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:45:10.252 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:10.249742 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:45:10.268 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:10.266090 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:45:10.269 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:10.266320 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:45:10.596 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:45:10.595775+0000 min lat: 0.00170736 max lat: 7.4213 avg lat: 0.01366 2022-03-08T18:45:10.596 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:45:10.597 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 20 16 23434 23418 73.1732 114.562 0.00657418 0.01366 2022-03-08T18:45:11.139 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:11.136873 I | op-mon: targeting the mon count 3 2022-03-08T18:45:11.165 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:11.161383 I | ceph-cluster-controller: Cluster update requested. CANCELLING CURRENT ORCHESTRATION 2022-03-08T18:45:11.165 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:11.161459 I | ceph-cluster-controller: reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:45:11.596 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 21 16 24540 24524 72.98 69.125 0.00269815 0.0135633 2022-03-08T18:45:11.752 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:11.749452 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:45:12.281 INFO:teuthology.orchestra.run.smithi002.stderr:+ '[' '' = 0 ']' 2022-03-08T18:45:12.282 INFO:teuthology.orchestra.run.smithi002.stderr:+ kubectl patch pv smithi066-lv-1 -p '{"spec":{"claimRef": null}}' 2022-03-08T18:45:12.596 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 22 16 24540 24524 69.6625 0 - 0.0135633 2022-03-08T18:45:12.765 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:12.763319 I | ceph-cluster-controller: detecting the ceph image version for image quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae... 2022-03-08T18:45:13.120 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:13.117976 I | op-k8sutil: Removing previous job rook-ceph-detect-version to start a new one 2022-03-08T18:45:13.493 INFO:teuthology.orchestra.run.smithi002.stdout:persistentvolume/smithi066-lv-1 patched 2022-03-08T18:45:13.499 INFO:teuthology.orchestra.run.smithi002.stderr:+ toolbox ceph orch apply osd --all-available-devices 2022-03-08T18:45:13.499 INFO:teuthology.orchestra.run.smithi002.stderr:+ kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph orch apply osd --all-available-devices 2022-03-08T18:45:13.596 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 23 16 24989 24973 67.8537 14.0312 0.00246111 0.0143614 2022-03-08T18:45:14.282 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:14.279904 I | op-k8sutil: batch job rook-ceph-detect-version still exists 2022-03-08T18:45:14.289 INFO:teuthology.orchestra.run.smithi002.stderr:Unable to use a TTY - input is not a terminal or the right kind of file 2022-03-08T18:45:14.596 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 24 16 24989 24973 65.0265 0 - 0.0143614 2022-03-08T18:45:15.597 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 25 16 24989 24973 62.4254 0 - 0.0143614 2022-03-08T18:45:16.391 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:16.388947 I | op-k8sutil: batch job rook-ceph-detect-version still exists 2022-03-08T18:45:16.597 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 26 16 24989 24973 60.0244 0 - 0.0143614 2022-03-08T18:45:17.598 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 27 16 24989 24973 57.8012 0 - 0.0143614 2022-03-08T18:45:18.507 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:18.504628 I | op-k8sutil: batch job rook-ceph-detect-version still exists 2022-03-08T18:45:18.597 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 28 16 24989 24973 55.737 0 - 0.0143614 2022-03-08T18:45:18.637 INFO:teuthology.orchestra.run.smithi002.stdout:['Success'] 2022-03-08T18:45:18.647 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:18.643587 I | ceph-cluster-controller: CR has changed for "rook-ceph". diff=  v1.ClusterSpec{ 2022-03-08T18:45:18.648 INFO:tasks.rook.operator.smithi002.stdout:   CephVersion: {Image: "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", AllowUnsupported: true}, 2022-03-08T18:45:18.648 INFO:tasks.rook.operator.smithi002.stdout:   Storage: v1.StorageScopeSpec{ 2022-03-08T18:45:18.648 INFO:tasks.rook.operator.smithi002.stdout:   ... // 3 identical fields 2022-03-08T18:45:18.648 INFO:tasks.rook.operator.smithi002.stdout:   Config: nil, 2022-03-08T18:45:18.649 INFO:tasks.rook.operator.smithi002.stdout:   Selection: {}, 2022-03-08T18:45:18.649 INFO:tasks.rook.operator.smithi002.stdout:   StorageClassDeviceSets: []v1.StorageClassDeviceSet{ 2022-03-08T18:45:18.649 INFO:tasks.rook.operator.smithi002.stdout:   ... // 9 identical elements 2022-03-08T18:45:18.649 INFO:tasks.rook.operator.smithi002.stdout:   {Name: "smithi137-lv-3", Count: 1, VolumeClaimTemplates: {{ObjectMeta: {Name: "data"}, Spec: {AccessModes: {"ReadWriteOnce"}, Resources: {Requests: {s"storage": {i: {...}, s: "1", Format: "DecimalSI"}}}, VolumeName: "smithi137-lv-3", StorageClassName: &"scratch", ...}}}}, 2022-03-08T18:45:18.650 INFO:tasks.rook.operator.smithi002.stdout:   {Name: "smithi137-lv-4", Count: 1, VolumeClaimTemplates: {{ObjectMeta: {Name: "data"}, Spec: {AccessModes: {"ReadWriteOnce"}, Resources: {Requests: {s"storage": {i: {...}, s: "1", Format: "DecimalSI"}}}, VolumeName: "smithi137-lv-4", StorageClassName: &"scratch", ...}}}}, 2022-03-08T18:45:18.650 INFO:tasks.rook.operator.smithi002.stdout:+  { 2022-03-08T18:45:18.650 INFO:tasks.rook.operator.smithi002.stdout:+  Name: "smithi066-lv-1", 2022-03-08T18:45:18.650 INFO:tasks.rook.operator.smithi002.stdout:+  Count: 1, 2022-03-08T18:45:18.651 INFO:tasks.rook.operator.smithi002.stdout:+  VolumeClaimTemplates: []v1.PersistentVolumeClaim{{ObjectMeta: v1.ObjectMeta{...}, Spec: v1.PersistentVolumeClaimSpec{...}}}, 2022-03-08T18:45:18.651 INFO:tasks.rook.operator.smithi002.stdout:+  }, 2022-03-08T18:45:18.651 INFO:tasks.rook.operator.smithi002.stdout:   }, 2022-03-08T18:45:18.651 INFO:tasks.rook.operator.smithi002.stdout:   }, 2022-03-08T18:45:18.651 INFO:tasks.rook.operator.smithi002.stdout:   Annotations: nil, 2022-03-08T18:45:18.652 INFO:tasks.rook.operator.smithi002.stdout:   Labels: nil, 2022-03-08T18:45:18.653 INFO:tasks.rook.operator.smithi002.stdout:   ... // 20 identical fields 2022-03-08T18:45:18.654 INFO:tasks.rook.operator.smithi002.stdout:  } 2022-03-08T18:45:18.664 INFO:teuthology.orchestra.run.smithi002.stderr:+ kubectl delete job rook-ceph-device-zap -n rook-ceph 2022-03-08T18:45:18.728 INFO:teuthology.orchestra.run.smithi002.stdout:job.batch "rook-ceph-device-zap" deleted 2022-03-08T18:45:18.751 INFO:teuthology.orchestra.run.smithi002.stderr:+ num_osd=0 2022-03-08T18:45:18.751 INFO:teuthology.orchestra.run.smithi002.stderr:+ '[' 0 '!=' 12 ']' 2022-03-08T18:45:18.751 INFO:teuthology.orchestra.run.smithi002.stderr:+ echo 'waiting for osd to come back up' 2022-03-08T18:45:18.752 INFO:teuthology.orchestra.run.smithi002.stdout:waiting for osd to come back up 2022-03-08T18:45:18.752 INFO:teuthology.orchestra.run.smithi002.stderr:++ toolbox ceph osd stat 2022-03-08T18:45:18.753 INFO:teuthology.orchestra.run.smithi002.stderr:++ kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd stat 2022-03-08T18:45:18.753 INFO:teuthology.orchestra.run.smithi002.stderr:++ cut -f3 -d ' ' 2022-03-08T18:45:18.889 INFO:teuthology.orchestra.run.smithi002.stderr:Unable to use a TTY - input is not a terminal or the right kind of file 2022-03-08T18:45:19.366 INFO:teuthology.orchestra.run.smithi002.stderr:+ num_osd=11 2022-03-08T18:45:19.367 INFO:teuthology.orchestra.run.smithi002.stderr:+ sleep 30s 2022-03-08T18:45:19.597 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 29 16 24989 24973 53.815 0 - 0.0143614 2022-03-08T18:45:20.512 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:20.510277 I | op-k8sutil: batch job rook-ceph-detect-version deleted 2022-03-08T18:45:20.597 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 30 16 24989 24973 52.0213 0 - 0.0143614 2022-03-08T18:45:21.597 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 31 16 24989 24973 50.3431 0 - 0.0143614 2022-03-08T18:45:22.598 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 32 16 24989 24973 48.7699 0 - 0.0143614 2022-03-08T18:45:23.598 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 33 16 24989 24973 47.292 0 - 0.0143614 2022-03-08T18:45:24.598 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 34 16 24989 24973 45.901 0 - 0.0143614 2022-03-08T18:45:25.598 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 35 16 24989 24973 44.5895 0 - 0.0143614 2022-03-08T18:45:26.598 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 36 16 24989 24973 43.3508 0 - 0.0143614 2022-03-08T18:45:27.598 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 37 16 25458 25442 42.9713 2.09375 0.00558413 0.0232654 2022-03-08T18:45:28.598 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 38 16 26919 26903 44.2432 91.3125 0.00795376 0.0225939 2022-03-08T18:45:29.599 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 39 16 28394 28378 45.4717 92.1875 0.0114562 0.0219843 2022-03-08T18:45:30.599 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:45:30.598725+0000 min lat: 0.00170736 max lat: 14.3006 avg lat: 0.0215511 2022-03-08T18:45:30.599 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:45:30.600 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 40 16 29708 29692 46.3878 82.125 0.013945 0.0215511 2022-03-08T18:45:31.599 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 41 16 31345 31329 47.7515 102.312 0.0084947 0.0209358 2022-03-08T18:45:32.599 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 42 16 32662 32646 48.5742 82.3125 0.00538765 0.0205601 2022-03-08T18:45:33.599 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 43 16 34041 34025 49.4486 86.1875 0.00472517 0.0202141 2022-03-08T18:45:34.599 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 44 16 35425 35409 50.2905 86.5 0.0068364 0.0198777 2022-03-08T18:45:35.600 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 45 16 36639 36623 50.8589 75.875 0.00346771 0.0195846 2022-03-08T18:45:36.600 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 46 16 36639 36623 49.7532 0 - 0.0195846 2022-03-08T18:45:37.600 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 47 16 37771 37755 50.1998 35.375 0.00692581 0.0199143 2022-03-08T18:45:38.600 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 48 16 38836 38820 50.5405 66.5625 0.00973128 0.0197802 2022-03-08T18:45:39.600 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 49 16 39845 39829 50.7958 63.0625 0.0135889 0.0196805 2022-03-08T18:45:40.600 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 50 16 40971 40955 51.1873 70.375 0.0116895 0.0195305 2022-03-08T18:45:41.600 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 51 16 42496 42480 52.0522 95.3125 0.00962114 0.0192055 2022-03-08T18:45:42.604 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 52 16 43796 43780 52.6094 81.25 0.00515251 0.0190017 2022-03-08T18:45:43.605 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 53 16 44806 44790 52.8077 63.125 0.0125 0.0189054 2022-03-08T18:45:44.605 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 54 16 46002 45986 53.2138 74.75 0.00527159 0.0187815 2022-03-08T18:45:45.605 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 55 16 47322 47306 53.746 82.5 0.0253069 0.0185986 2022-03-08T18:45:46.605 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 56 16 48686 48670 54.308 85.25 0.00557775 0.0184072 2022-03-08T18:45:47.605 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 57 16 49343 49327 54.0755 41.0625 0.0142996 0.0184866 2022-03-08T18:45:48.605 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 58 16 50617 50601 54.5159 79.625 0.00619456 0.018337 2022-03-08T18:45:49.367 INFO:teuthology.orchestra.run.smithi002.stderr:+ '[' 11 '!=' 12 ']' 2022-03-08T18:45:49.368 INFO:teuthology.orchestra.run.smithi002.stdout:waiting for osd to come back up 2022-03-08T18:45:49.368 INFO:teuthology.orchestra.run.smithi002.stderr:+ echo 'waiting for osd to come back up' 2022-03-08T18:45:49.369 INFO:teuthology.orchestra.run.smithi002.stderr:++ toolbox ceph osd stat 2022-03-08T18:45:49.369 INFO:teuthology.orchestra.run.smithi002.stderr:++ kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd stat 2022-03-08T18:45:49.370 INFO:teuthology.orchestra.run.smithi002.stderr:++ cut -f3 -d ' ' 2022-03-08T18:45:49.606 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 59 16 51978 51962 55.0334 85.0625 0.00865946 0.0181633 2022-03-08T18:45:49.916 INFO:teuthology.orchestra.run.smithi002.stderr:Unable to use a TTY - input is not a terminal or the right kind of file 2022-03-08T18:45:50.606 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:45:50.605555+0000 min lat: 0.00170736 max lat: 14.3006 avg lat: 0.0181233 2022-03-08T18:45:50.606 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:45:50.606 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 60 16 52979 52963 55.1588 62.5625 0.00925751 0.0181233 2022-03-08T18:45:51.606 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 61 16 53550 53534 54.8395 35.6875 0.0027165 0.0181502 2022-03-08T18:45:52.606 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 62 16 53996 53980 54.4046 27.875 0.0593436 0.0182585 2022-03-08T18:45:53.606 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 63 15 54126 54111 53.671 8.1875 0.00733885 0.0186244 2022-03-08T18:45:53.826 INFO:teuthology.orchestra.run.smithi002.stderr:+ num_osd=11 2022-03-08T18:45:53.826 INFO:teuthology.orchestra.run.smithi002.stderr:+ sleep 30s 2022-03-08T18:45:54.606 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 64 16 55269 55253 53.9475 71.375 0.0077535 0.0185306 2022-03-08T18:45:55.606 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 65 16 56552 56536 54.3511 80.1875 0.00910107 0.0183905 2022-03-08T18:45:56.606 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 66 16 57665 57649 54.5814 69.5625 0.0116024 0.0183153 2022-03-08T18:45:57.335 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:57.332769 I | ceph-cluster-controller: detected ceph image version: "17.0.0-10961 quincy" 2022-03-08T18:45:57.336 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:57.332795 I | ceph-cluster-controller: validating ceph version from provided image 2022-03-08T18:45:57.336 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:57.332807 W | ceph-cluster-controller: unsupported ceph version detected: "17.0.0-10961 quincy", pursuing 2022-03-08T18:45:57.346 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:57.343116 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:45:57.427 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:57.424931 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:45:57.427 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:57.425126 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:45:57.607 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 67 16 58794 58778 54.8198 70.5625 0.0103239 0.0182346 2022-03-08T18:45:57.992 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:57.990431 I | ceph-cluster-controller: cluster "rook-ceph": version "17.0.0-10961 quincy" detected for image "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae" 2022-03-08T18:45:58.206 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:58.203816 I | op-mon: start running mons 2022-03-08T18:45:58.230 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:58.228262 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:45:58.245 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:58.243416 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:45:58.325 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:58.322964 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:45:58.325 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:58.323122 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:45:58.607 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 68 16 59513 59497 54.6744 44.9375 0.00388485 0.0182711 2022-03-08T18:45:59.280 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:59.277111 I | op-mon: targeting the mon count 3 2022-03-08T18:45:59.299 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:59.297548 I | ceph-cluster-controller: Cluster update requested. CANCELLING CURRENT ORCHESTRATION 2022-03-08T18:45:59.300 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:59.297650 I | ceph-cluster-controller: reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:45:59.607 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 69 16 60686 60670 54.9444 73.3125 0.0107871 0.0181938 2022-03-08T18:45:59.856 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:45:59.852741 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:46:00.276 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:00.274115 I | ceph-cluster-controller: detecting the ceph image version for image quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae... 2022-03-08T18:46:00.607 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 70 16 61075 61059 54.5068 24.3125 0.00444105 0.0181616 2022-03-08T18:46:00.668 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:00.663819 I | op-k8sutil: Removing previous job rook-ceph-detect-version to start a new one 2022-03-08T18:46:01.427 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:01.424672 I | op-k8sutil: batch job rook-ceph-detect-version still exists 2022-03-08T18:46:01.607 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 71 16 61428 61412 54.0498 22.0625 0.0123174 0.0184934 2022-03-08T18:46:02.607 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 72 16 62672 62656 54.3788 77.75 0.00963026 0.0183836 2022-03-08T18:46:03.450 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:03.448082 I | op-k8sutil: batch job rook-ceph-detect-version still exists 2022-03-08T18:46:03.607 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 73 16 63788 63772 54.5893 69.75 0.0165831 0.0183124 2022-03-08T18:46:04.607 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 74 16 64949 64933 54.8321 72.5625 0.0087678 0.0182318 2022-03-08T18:46:05.458 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:05.455879 I | op-k8sutil: batch job rook-ceph-detect-version deleted 2022-03-08T18:46:05.607 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 75 16 66111 66095 55.0692 72.625 0.00445758 0.0181515 2022-03-08T18:46:06.608 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 76 16 67268 67252 55.2959 72.3125 0.0243734 0.0180786 2022-03-08T18:46:07.608 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 77 16 68046 68030 55.2093 48.625 0.00464346 0.0181061 2022-03-08T18:46:08.608 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 78 16 69185 69169 55.414 71.1875 0.0112125 0.0180403 2022-03-08T18:46:09.608 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 79 16 69594 69578 55.0361 25.5625 0.00759493 0.018024 2022-03-08T18:46:10.608 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:46:10.607742+0000 min lat: 0.00170736 max lat: 14.3006 avg lat: 0.0182115 2022-03-08T18:46:10.608 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:46:10.609 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 80 16 70288 70272 54.8903 43.375 0.0102555 0.0182115 2022-03-08T18:46:11.608 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 81 16 71344 71328 55.0273 66 0.0140554 0.0181665 2022-03-08T18:46:12.609 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 82 16 71827 71811 54.724 30.1875 0.0113404 0.0182675 2022-03-08T18:46:13.609 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 83 16 72956 72940 54.9147 70.5625 0.0141048 0.0182043 2022-03-08T18:46:14.609 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 84 16 73458 73442 54.6345 31.375 0.00657724 0.0181789 2022-03-08T18:46:15.236 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:15.230383 I | ceph-cluster-controller: detected ceph image version: "17.0.0-10961 quincy" 2022-03-08T18:46:15.237 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:15.230406 I | ceph-cluster-controller: validating ceph version from provided image 2022-03-08T18:46:15.237 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:15.230417 W | ceph-cluster-controller: unsupported ceph version detected: "17.0.0-10961 quincy", pursuing 2022-03-08T18:46:15.245 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:15.241338 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:46:15.264 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:15.260751 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:46:15.265 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:15.260961 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:46:15.609 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 85 16 73856 73840 54.2843 24.875 0.00417792 0.0183028 2022-03-08T18:46:15.725 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:15.723528 I | ceph-cluster-controller: cluster "rook-ceph": version "17.0.0-10961 quincy" detected for image "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae" 2022-03-08T18:46:15.887 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:15.883253 I | op-mon: start running mons 2022-03-08T18:46:15.916 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:15.914054 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:46:15.927 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:15.925237 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:c=10.109.10.123:6789,a=10.101.136.169:6789,b=10.110.64.66:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:46:15.940 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:15.938717 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:46:15.944 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:15.938892 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:46:16.610 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 86 16 73856 73840 53.6532 0 - 0.0183028 2022-03-08T18:46:17.016 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:17.014188 I | op-mon: targeting the mon count 3 2022-03-08T18:46:17.025 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:17.021068 I | op-config: setting "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:46:17.491 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:17.488620 I | op-config: successfully set "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:46:17.491 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:17.488647 I | op-config: setting "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:46:17.611 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 87 16 74498 74482 53.4976 20.0625 0.00927056 0.0186852 2022-03-08T18:46:18.016 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:18.013438 I | op-config: successfully set "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:46:18.016 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:18.013512 I | op-config: setting "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:46:18.609 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 88 16 75520 75504 53.6155 63.875 0.00832832 0.0186455 2022-03-08T18:46:18.978 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:18.972573 I | op-config: successfully set "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:46:18.979 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:18.972600 I | op-config: setting "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:46:19.485 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:19.482444 I | op-config: successfully set "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:46:19.485 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:19.482472 I | op-config: setting "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:46:19.610 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 89 16 76234 76218 53.5144 44.625 0.00541083 0.0186261 2022-03-08T18:46:19.939 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:19.937631 I | op-config: successfully set "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:46:19.940 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:19.937653 I | op-config: deleting "log file" option from the mon configuration database 2022-03-08T18:46:20.458 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:20.455949 I | op-config: successfully deleted "log file" option from the mon configuration database 2022-03-08T18:46:20.458 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:20.455967 I | op-mon: checking for basic quorum with existing mons 2022-03-08T18:46:20.532 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:20.530297 I | op-mon: mon "b" endpoint is [v2:10.110.64.66:3300,v1:10.110.64.66:6789] 2022-03-08T18:46:20.610 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 90 16 77293 77277 53.6551 66.1875 0.0131643 0.0186313 2022-03-08T18:46:20.651 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:20.648896 I | op-mon: mon "c" endpoint is [v2:10.109.10.123:3300,v1:10.109.10.123:6789] 2022-03-08T18:46:20.865 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:20.861708 I | op-mon: mon "a" endpoint is [v2:10.101.136.169:3300,v1:10.101.136.169:6789] 2022-03-08T18:46:21.488 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:21.484259 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:46:21.610 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 91 16 78343 78327 53.7866 65.625 0.0133211 0.0185847 2022-03-08T18:46:22.062 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:22.060618 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:46:22.063 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:22.060842 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:46:22.483 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:22.481252 I | op-mon: deployment for mon rook-ceph-mon-b already exists. updating if needed 2022-03-08T18:46:22.493 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:22.491426 I | op-k8sutil: deployment "rook-ceph-mon-b" did not change, nothing to update 2022-03-08T18:46:22.493 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:22.491452 I | op-mon: waiting for mon quorum with [b c a] 2022-03-08T18:46:22.610 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 92 16 78598 78582 53.3752 15.9375 0.00522769 0.0185526 2022-03-08T18:46:23.069 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:23.067247 I | op-mon: mons running: [b c a] 2022-03-08T18:46:23.610 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 93 16 79324 79308 53.2891 45.375 0.00635084 0.0187591 2022-03-08T18:46:23.612 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:23.609933 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:46:23.618 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:23.616296 I | op-mon: deployment for mon rook-ceph-mon-c already exists. updating if needed 2022-03-08T18:46:23.627 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:23.624928 I | op-k8sutil: deployment "rook-ceph-mon-c" did not change, nothing to update 2022-03-08T18:46:23.627 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:23.624956 I | op-mon: waiting for mon quorum with [b c a] 2022-03-08T18:46:23.669 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:23.667241 I | op-mon: mons running: [b c a] 2022-03-08T18:46:23.827 INFO:teuthology.orchestra.run.smithi002.stderr:+ '[' 11 '!=' 12 ']' 2022-03-08T18:46:23.827 INFO:teuthology.orchestra.run.smithi002.stderr:+ echo 'waiting for osd to come back up' 2022-03-08T18:46:23.828 INFO:teuthology.orchestra.run.smithi002.stdout:waiting for osd to come back up 2022-03-08T18:46:23.828 INFO:teuthology.orchestra.run.smithi002.stderr:++ toolbox ceph osd stat 2022-03-08T18:46:23.829 INFO:teuthology.orchestra.run.smithi002.stderr:++ kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd stat 2022-03-08T18:46:23.829 INFO:teuthology.orchestra.run.smithi002.stderr:++ cut -f3 -d ' ' 2022-03-08T18:46:23.946 INFO:teuthology.orchestra.run.smithi002.stderr:Unable to use a TTY - input is not a terminal or the right kind of file 2022-03-08T18:46:24.284 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:24.282683 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:46:24.290 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:24.288380 I | op-mon: deployment for mon rook-ceph-mon-a already exists. updating if needed 2022-03-08T18:46:24.304 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:24.302385 I | op-k8sutil: deployment "rook-ceph-mon-a" did not change, nothing to update 2022-03-08T18:46:24.305 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:24.302417 I | op-mon: waiting for mon quorum with [b c a] 2022-03-08T18:46:24.335 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:24.332919 I | op-mon: mons running: [b c a] 2022-03-08T18:46:24.472 INFO:teuthology.orchestra.run.smithi002.stderr:+ num_osd=11 2022-03-08T18:46:24.473 INFO:teuthology.orchestra.run.smithi002.stderr:+ sleep 30s 2022-03-08T18:46:24.610 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 94 16 80028 80012 53.1903 44 0.00824034 0.0187221 2022-03-08T18:46:24.771 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:24.768882 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:46:24.772 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:24.768906 I | op-mon: mons created: 3 2022-03-08T18:46:25.184 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:25.182184 I | op-mon: waiting for mon quorum with [b c a] 2022-03-08T18:46:25.610 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 95 16 80058 80042 52.6502 1.875 0.00585251 0.0187173 2022-03-08T18:46:26.170 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:26.167675 I | op-mon: mons running: [b c a] 2022-03-08T18:46:26.584 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:26.581565 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:46:26.584 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:26.581611 I | cephclient: getting or creating ceph auth key "client.csi-rbd-provisioner" 2022-03-08T18:46:26.610 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 96 16 80058 80042 52.1017 0 - 0.0187173 2022-03-08T18:46:27.007 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:27.005245 I | cephclient: getting or creating ceph auth key "client.csi-rbd-node" 2022-03-08T18:46:27.424 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:27.421528 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-provisioner" 2022-03-08T18:46:27.610 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 97 16 80058 80042 51.5646 0 - 0.0187173 2022-03-08T18:46:27.830 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:27.828408 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-node" 2022-03-08T18:46:28.611 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 98 16 80058 80042 51.0385 0 - 0.0187173 2022-03-08T18:46:29.611 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 99 16 80058 80042 50.523 0 - 0.0187173 2022-03-08T18:46:29.686 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:29.683913 I | ceph-csi: created kubernetes csi secrets for cluster "rook-ceph" 2022-03-08T18:46:29.686 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:29.683941 I | cephclient: getting or creating ceph auth key "client.crash" 2022-03-08T18:46:30.105 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:30.102926 I | ceph-crashcollector-controller: created kubernetes crash collector secret for cluster "rook-ceph" 2022-03-08T18:46:30.522 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:30.519567 I | cephclient: successfully enabled msgr2 protocol 2022-03-08T18:46:30.522 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:30.519601 I | cephclient: create rbd-mirror bootstrap peer token "client.rbd-mirror-peer" 2022-03-08T18:46:30.522 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:30.519609 I | cephclient: getting or creating ceph auth key "client.rbd-mirror-peer" 2022-03-08T18:46:30.611 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:46:30.610570+0000 min lat: 0.00170736 max lat: 14.3006 avg lat: 0.0187173 2022-03-08T18:46:30.611 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:46:30.612 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 100 16 80058 80042 50.0178 0 - 0.0187173 2022-03-08T18:46:30.952 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:30.949592 I | cephclient: successfully created rbd-mirror bootstrap peer token for cluster "rook-ceph" 2022-03-08T18:46:31.017 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.013021 I | ceph-spec: controller will reconcile resource "cluster-peer-token-rook-ceph" based on patch: [redacted patch details due to potentially sensitive content] 2022-03-08T18:46:31.039 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.036580 I | op-mgr: start running mgr 2022-03-08T18:46:31.039 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.036616 I | cephclient: getting or creating ceph auth key "mgr.a" 2022-03-08T18:46:31.504 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.502137 I | op-mgr: deployment for mgr rook-ceph-mgr-a already exists. updating if needed 2022-03-08T18:46:31.513 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.511429 I | op-k8sutil: deployment "rook-ceph-mgr-a" did not change, nothing to update 2022-03-08T18:46:31.516 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.513996 I | op-mgr: setting services to point to mgr "a" 2022-03-08T18:46:31.611 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 101 16 80058 80042 49.5225 0 - 0.0187173 2022-03-08T18:46:31.676 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.672962 I | op-osd: start running osds in namespace "rook-ceph" 2022-03-08T18:46:31.677 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.672998 I | op-osd: wait timeout for healthy OSDs during upgrade or restart is "10m0s" 2022-03-08T18:46:31.809 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.806277 I | op-osd: start provisioning the OSDs on PVCs, if needed 2022-03-08T18:46:31.816 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811848 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-1" 2022-03-08T18:46:31.816 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811875 I | op-osd: OSD PVC "smithi002-lv-1-data-0cg8pd" already exists 2022-03-08T18:46:31.817 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811888 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-2" 2022-03-08T18:46:31.817 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811899 I | op-osd: OSD PVC "smithi002-lv-2-data-09sjq5" already exists 2022-03-08T18:46:31.818 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811906 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-3" 2022-03-08T18:46:31.818 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811914 I | op-osd: OSD PVC "smithi002-lv-3-data-05dmhp" already exists 2022-03-08T18:46:31.818 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811929 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-4" 2022-03-08T18:46:31.819 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811938 I | op-osd: OSD PVC "smithi002-lv-4-data-0pb5hz" already exists 2022-03-08T18:46:31.819 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811944 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-2" 2022-03-08T18:46:31.819 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811953 I | op-osd: OSD PVC "smithi066-lv-2-data-0jsgjv" already exists 2022-03-08T18:46:31.820 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811960 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-3" 2022-03-08T18:46:31.820 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811968 I | op-osd: OSD PVC "smithi066-lv-3-data-0xl876" already exists 2022-03-08T18:46:31.820 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811979 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-4" 2022-03-08T18:46:31.820 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.811993 I | op-osd: OSD PVC "smithi066-lv-4-data-04lwmk" already exists 2022-03-08T18:46:31.820 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.812000 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-1" 2022-03-08T18:46:31.821 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.812008 I | op-osd: OSD PVC "smithi137-lv-1-data-0cmhmp" already exists 2022-03-08T18:46:31.821 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.812018 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-2" 2022-03-08T18:46:31.821 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.812025 I | op-osd: OSD PVC "smithi137-lv-2-data-0whzhj" already exists 2022-03-08T18:46:31.822 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.812033 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-3" 2022-03-08T18:46:31.822 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.812042 I | op-osd: OSD PVC "smithi137-lv-3-data-0kg6xk" already exists 2022-03-08T18:46:31.822 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.812048 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-4" 2022-03-08T18:46:31.822 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.812055 I | op-osd: OSD PVC "smithi137-lv-4-data-0zchnd" already exists 2022-03-08T18:46:31.823 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.812061 I | op-osd: creating 1 new PVCs for device set "smithi066-lv-1" 2022-03-08T18:46:31.868 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.865453 I | op-osd: successfully provisioned PVC "smithi066-lv-1-data-0snd8j" 2022-03-08T18:46:31.904 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.900392 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:46:31.905 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.900423 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-2-data-09sjq5" 2022-03-08T18:46:31.905 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.900432 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-3-data-05dmhp" 2022-03-08T18:46:31.906 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.900445 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:46:31.906 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.900451 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:46:31.906 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.900457 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-3-data-0xl876" 2022-03-08T18:46:31.906 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.900463 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-4-data-04lwmk" 2022-03-08T18:46:31.906 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.900473 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:46:31.907 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.900479 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-2-data-0whzhj" 2022-03-08T18:46:31.907 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.900484 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:46:31.907 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.900490 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-4-data-0zchnd" 2022-03-08T18:46:31.907 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:31.900496 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-1-data-0snd8j" 2022-03-08T18:46:32.285 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:32.282909 I | op-osd: started OSD provisioning job for PVC "smithi066-lv-1-data-0snd8j" 2022-03-08T18:46:32.286 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:32.282930 I | op-osd: start provisioning the OSDs on nodes, if needed 2022-03-08T18:46:32.286 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:32.282936 I | op-osd: no nodes are defined for configuring OSDs on raw devices 2022-03-08T18:46:32.467 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:32.464709 I | op-osd: OSD orchestration status for PVC smithi066-lv-1-data-0snd8j is "starting" 2022-03-08T18:46:32.612 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 102 16 80058 80042 49.037 0 - 0.0187173 2022-03-08T18:46:33.119 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:33.117151 I | op-mgr: successful modules: mgr module(s) from the spec 2022-03-08T18:46:33.126 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:33.123814 I | op-mgr: successful modules: dashboard 2022-03-08T18:46:33.126 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:33.123852 I | op-mgr: successful modules: prometheus 2022-03-08T18:46:33.611 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 103 16 80058 80042 48.561 0 - 0.0187173 2022-03-08T18:46:33.629 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:33.626613 I | op-mgr: successful modules: balancer 2022-03-08T18:46:33.955 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:33.952526 I | op-osd: updating OSD 1 on PVC "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:46:33.956 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:33.952553 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:46:34.081 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:34.079175 I | op-osd: updating OSD 7 on PVC "smithi137-lv-2-data-0whzhj" 2022-03-08T18:46:34.082 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:34.079262 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-2-data-0whzhj" 2022-03-08T18:46:34.578 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:34.576211 I | op-mgr: successful modules: orchestrator modules 2022-03-08T18:46:34.612 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 104 16 80058 80042 48.0941 0 - 0.0187173 2022-03-08T18:46:35.604 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:35.601829 I | op-osd: updating OSD 10 on PVC "smithi137-lv-4-data-0zchnd" 2022-03-08T18:46:35.604 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:35.601855 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-4-data-0zchnd" 2022-03-08T18:46:35.612 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 105 16 80058 80042 47.636 0 - 0.0187173 2022-03-08T18:46:36.612 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 106 16 80058 80042 47.1867 0 - 0.0187173 2022-03-08T18:46:37.147 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:37.144279 I | op-osd: updating OSD 5 on PVC "smithi002-lv-2-data-09sjq5" 2022-03-08T18:46:37.147 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:37.144299 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-2-data-09sjq5" 2022-03-08T18:46:37.199 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:37.197223 I | op-osd: updating OSD 11 on PVC "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:46:37.200 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:37.197258 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:46:37.612 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 107 16 80058 80042 46.7457 0 - 0.0187173 2022-03-08T18:46:38.617 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 108 16 80058 80042 46.3107 0 - 0.0187173 2022-03-08T18:46:38.742 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:38.740147 I | op-osd: updating OSD 2 on PVC "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:46:38.743 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:38.740180 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:46:39.617 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 109 16 80344 80328 46.0498 1.27679 0.00649843 0.0217057 2022-03-08T18:46:40.618 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 110 16 81090 81074 46.055 46.625 0.0078973 0.0217064 2022-03-08T18:46:40.646 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:40.644761 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:46:41.618 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 111 16 82146 82130 46.2346 66 0.00796196 0.0216223 2022-03-08T18:46:42.618 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 112 15 83093 83078 46.3507 59.25 0.0283558 0.0215679 2022-03-08T18:46:42.959 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:42.956380 I | op-osd: updating OSD 4 on PVC "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:46:42.959 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:42.956401 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:46:43.619 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 113 16 84085 84069 46.4886 61.9375 0.00714925 0.0215047 2022-03-08T18:46:44.618 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 114 16 85100 85084 46.6372 63.4375 0.0113872 0.0214359 2022-03-08T18:46:45.504 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:45.501782 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:46:45.618 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 115 16 86003 85987 46.7223 56.4375 0.00623494 0.0213687 2022-03-08T18:46:46.618 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 116 16 86003 85987 46.3196 0 - 0.0213687 2022-03-08T18:46:47.222 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:47.218485 I | op-osd: updating OSD 8 on PVC "smithi002-lv-3-data-05dmhp" 2022-03-08T18:46:47.223 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:47.218515 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-3-data-05dmhp" 2022-03-08T18:46:47.619 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 117 16 86819 86803 46.3593 25.5 0.00702261 0.0215642 2022-03-08T18:46:48.619 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 118 16 87883 87867 46.5299 66.5 0.0100767 0.0214853 2022-03-08T18:46:49.619 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 119 16 88873 88857 46.6588 61.875 0.011076 0.021426 2022-03-08T18:46:49.682 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:49.679982 I | op-osd: updating OSD 9 on PVC "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:46:49.683 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:49.680012 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:46:50.619 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:46:50.618742+0000 min lat: 0.00170736 max lat: 15.2919 avg lat: 0.0213604 2022-03-08T18:46:50.620 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:46:50.620 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 120 16 89892 89876 46.8006 63.6875 0.00515256 0.0213604 2022-03-08T18:46:51.620 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 121 16 90906 90890 46.9375 63.375 0.0138031 0.0212983 2022-03-08T18:46:51.729 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:51.725074 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:46:52.619 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 122 16 91896 91880 47.0599 61.875 0.00857222 0.0212438 2022-03-08T18:46:53.618 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:53.615169 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:46:53.619 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 123 16 92847 92831 47.1605 59.4375 0.0162121 0.0211984 2022-03-08T18:46:54.474 INFO:teuthology.orchestra.run.smithi002.stderr:+ '[' 11 '!=' 12 ']' 2022-03-08T18:46:54.474 INFO:teuthology.orchestra.run.smithi002.stdout:waiting for osd to come back up 2022-03-08T18:46:54.475 INFO:teuthology.orchestra.run.smithi002.stderr:+ echo 'waiting for osd to come back up' 2022-03-08T18:46:54.480 INFO:teuthology.orchestra.run.smithi002.stderr:++ toolbox ceph osd stat 2022-03-08T18:46:54.480 INFO:teuthology.orchestra.run.smithi002.stderr:++ kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd stat 2022-03-08T18:46:54.481 INFO:teuthology.orchestra.run.smithi002.stderr:++ cut -f3 -d ' ' 2022-03-08T18:46:54.596 INFO:teuthology.orchestra.run.smithi002.stderr:Unable to use a TTY - input is not a terminal or the right kind of file 2022-03-08T18:46:54.620 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 124 16 93785 93769 47.2529 58.625 0.0154594 0.0211572 2022-03-08T18:46:55.309 INFO:teuthology.orchestra.run.smithi002.stderr:+ num_osd=11 2022-03-08T18:46:55.309 INFO:teuthology.orchestra.run.smithi002.stderr:+ sleep 30s 2022-03-08T18:46:55.579 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:55.576436 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:46:55.620 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 125 16 94746 94730 47.3553 60.0625 0.0275464 0.0211102 2022-03-08T18:46:56.620 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 126 16 95705 95689 47.4551 59.9375 0.0102926 0.021066 2022-03-08T18:46:57.518 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:57.516399 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:46:57.519 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:57.516462 I | op-osd: OSD orchestration status for node smithi066-lv-1-data-0snd8j is "orchestrating" 2022-03-08T18:46:57.521 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:57.516650 I | op-osd: OSD orchestration status for PVC smithi066-lv-1-data-0snd8j is "orchestrating" 2022-03-08T18:46:57.620 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 127 16 96572 96556 47.508 54.1875 0.0231767 0.0210426 2022-03-08T18:46:58.620 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 128 16 97510 97494 47.5948 58.625 0.0104231 0.0210039 2022-03-08T18:46:59.478 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:46:59.475690 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:46:59.620 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 129 16 98369 98353 47.642 53.6875 0.00993357 0.0209836 2022-03-08T18:47:00.621 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 130 16 98915 98899 47.538 34.125 0.0359095 0.0210288 2022-03-08T18:47:01.342 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:01.339732 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:01.622 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 131 16 99859 99843 47.6255 59 0.0183054 0.0209907 2022-03-08T18:47:02.621 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 132 16 100816 100800 47.7177 59.8125 0.0133717 0.0209498 2022-03-08T18:47:03.270 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:03.268212 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:03.621 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 133 16 101629 101613 47.741 50.8125 0.011156 0.0209399 2022-03-08T18:47:04.621 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 134 16 102285 102269 47.6906 41 0.00858266 0.0209496 2022-03-08T18:47:05.474 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:05.471308 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:05.474 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:05.471407 I | op-osd: OSD orchestration status for PVC smithi066-lv-1-data-0snd8j is "completed" 2022-03-08T18:47:05.474 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:05.471424 I | op-osd: creating OSD 0 on PVC "smithi066-lv-1-data-0snd8j" 2022-03-08T18:47:05.474 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:05.471437 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-1-data-0snd8j" 2022-03-08T18:47:05.621 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 135 16 103033 103017 47.6836 46.75 0.0418544 0.0209646 2022-03-08T18:47:06.621 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 136 16 103794 103778 47.6827 47.5625 0.0233253 0.0209653 2022-03-08T18:47:07.537 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:07.535153 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:07.621 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 137 16 104596 104580 47.7005 50.125 0.0173095 0.0209578 2022-03-08T18:47:08.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 138 16 105321 105305 47.6831 45.3125 0.00510705 0.0209329 2022-03-08T18:47:09.354 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:09.352066 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:09.621 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 139 16 105866 105850 47.5851 34.0625 0.011597 0.0210087 2022-03-08T18:47:10.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:47:10.620798+0000 min lat: 0.00170736 max lat: 15.2919 avg lat: 0.0209873 2022-03-08T18:47:10.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:47:10.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 140 16 106736 106720 47.6336 54.375 0.029581 0.0209873 2022-03-08T18:47:11.317 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:11.314660 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:11.622 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 141 16 107569 107553 47.665 52.0625 0.00710004 0.0209728 2022-03-08T18:47:12.622 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 142 16 108471 108455 47.7262 56.375 0.0156085 0.0209464 2022-03-08T18:47:13.622 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 143 16 109348 109332 47.7758 54.8125 0.0103651 0.0209249 2022-03-08T18:47:13.716 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:13.713234 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:14.622 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 144 16 110132 110116 47.7842 49 0.0104422 0.020921 2022-03-08T18:47:15.566 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:15.563744 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:15.622 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 145 16 110717 110701 47.7068 36.5625 0.0127628 0.020955 2022-03-08T18:47:16.622 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 146 16 111603 111587 47.7593 55.375 0.0103651 0.0209326 2022-03-08T18:47:17.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 147 16 112452 112436 47.7953 53.0625 0.00735915 0.0209153 2022-03-08T18:47:17.727 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:17.725419 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:18.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 148 16 113291 113275 47.8267 52.4375 0.00989838 0.0209026 2022-03-08T18:47:19.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 149 16 114135 114119 47.8597 52.75 0.00768021 0.0208875 2022-03-08T18:47:20.608 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:20.606615 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:20.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 150 16 114979 114963 47.8922 52.75 0.0308699 0.0208741 2022-03-08T18:47:21.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 151 16 115842 115826 47.9322 53.9375 0.0210752 0.0208564 2022-03-08T18:47:22.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 152 16 116710 116694 47.9738 54.25 0.0164485 0.0208386 2022-03-08T18:47:23.160 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:23.158052 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:23.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 153 16 117503 117487 47.9841 49.5625 0.0312992 0.020833 2022-03-08T18:47:24.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 154 16 118278 118262 47.987 48.4375 0.0094325 0.0208327 2022-03-08T18:47:25.310 INFO:teuthology.orchestra.run.smithi002.stderr:+ '[' 11 '!=' 12 ']' 2022-03-08T18:47:25.311 INFO:teuthology.orchestra.run.smithi002.stderr:+ echo 'waiting for osd to come back up' 2022-03-08T18:47:25.312 INFO:teuthology.orchestra.run.smithi002.stdout:waiting for osd to come back up 2022-03-08T18:47:25.312 INFO:teuthology.orchestra.run.smithi002.stderr:++ cut -f3 -d ' ' 2022-03-08T18:47:25.313 INFO:teuthology.orchestra.run.smithi002.stderr:++ toolbox ceph osd stat 2022-03-08T18:47:25.313 INFO:teuthology.orchestra.run.smithi002.stderr:++ kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd stat 2022-03-08T18:47:25.404 INFO:teuthology.orchestra.run.smithi002.stderr:Unable to use a TTY - input is not a terminal or the right kind of file 2022-03-08T18:47:25.430 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:25.427484 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:25.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 155 16 119065 119049 47.9947 49.1875 0.0261893 0.0208292 2022-03-08T18:47:26.623 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 156 15 119848 119833 48.0011 49 0.0212783 0.0208273 2022-03-08T18:47:27.000 INFO:teuthology.orchestra.run.smithi002.stderr:+ num_osd=11 2022-03-08T18:47:27.001 INFO:teuthology.orchestra.run.smithi002.stderr:+ sleep 30s 2022-03-08T18:47:27.624 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 157 16 120490 120474 47.9506 40.0625 0.00858748 0.0208489 2022-03-08T18:47:28.024 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:28.022436 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:28.624 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 158 16 121263 121247 47.9528 48.3125 0.0129896 0.0208479 2022-03-08T18:47:29.624 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 159 16 122008 121992 47.944 46.5625 0.0183199 0.0208515 2022-03-08T18:47:30.288 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:30.286500 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:30.624 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:47:30.622804+0000 min lat: 0.00170736 max lat: 15.2919 avg lat: 0.0208749 2022-03-08T18:47:30.941 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:47:30.942 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 160 16 122635 122619 47.8893 39.1875 0.00935651 0.0208749 2022-03-08T18:47:31.624 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 161 16 123166 123150 47.798 33.1875 0.0326067 0.020915 2022-03-08T18:47:32.284 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:32.282037 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:32.287 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:32.283015 I | op-osd: waiting... 1 of 1 OSD prepare jobs have finished processing and 9 of 11 OSDs have been updated 2022-03-08T18:47:32.624 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 162 16 123969 123953 47.8127 50.1875 0.0147238 0.0209078 2022-03-08T18:47:33.624 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 163 16 124720 124704 47.8073 46.9375 0.02067 0.0209111 2022-03-08T18:47:34.459 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:34.456666 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:34.624 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 164 16 125486 125470 47.8077 47.875 0.022727 0.020911 2022-03-08T18:47:35.625 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 165 16 125987 125971 47.7077 31.3125 0.0081452 0.0209086 2022-03-08T18:47:36.516 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:36.513582 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:36.624 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 166 16 125987 125971 47.4203 0 - 0.0209086 2022-03-08T18:47:37.625 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 167 16 125987 125971 47.1364 0 - 0.0209086 2022-03-08T18:47:38.094 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:38.092389 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:38.625 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 168 16 126013 125997 46.8655 0.541667 0.00713161 0.020969 2022-03-08T18:47:39.625 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 169 16 126013 125997 46.5882 0 - 0.020969 2022-03-08T18:47:39.796 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:39.793602 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:40.626 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 170 16 126419 126403 46.4632 12.6875 0.0141758 0.0215137 2022-03-08T18:47:41.626 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 171 16 126719 126703 46.3011 18.75 0.372795 0.021589 2022-03-08T18:47:42.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:42.085439 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:42.626 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 172 16 127202 127186 46.2074 30.1875 0.0233646 0.0216325 2022-03-08T18:47:43.626 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 173 16 127392 127376 46.009 11.875 0.0144548 0.0216917 2022-03-08T18:47:43.913 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:43.910850 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:44.627 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 174 16 127850 127834 45.9091 28.625 0.0437622 0.0217734 2022-03-08T18:47:45.627 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 175 16 128314 128298 45.8124 29 0.00928172 0.0218204 2022-03-08T18:47:45.711 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:45.708919 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:46.627 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 176 16 128708 128692 45.692 24.625 0.0111106 0.021867 2022-03-08T18:47:47.528 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:47.525327 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:47.627 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 177 16 129192 129176 45.6048 30.25 0.0112624 0.0219198 2022-03-08T18:47:48.627 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 178 16 129506 129490 45.4588 19.625 0.0113079 0.0219537 2022-03-08T18:47:49.352 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:49.349642 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:49.627 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 179 16 129978 129962 45.3697 29.5 0.037723 0.0220336 2022-03-08T18:47:50.627 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:47:50.625741+0000 min lat: 0.00170736 max lat: 15.2919 avg lat: 0.0220752 2022-03-08T18:47:50.628 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:47:50.628 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 180 16 130422 130406 45.2718 27.75 0.0240292 0.0220752 2022-03-08T18:47:51.209 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:51.207236 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:51.627 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 181 16 130689 130673 45.1138 16.6875 0.0125335 0.0221586 2022-03-08T18:47:52.628 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 182 16 131189 131173 45.0377 31.25 0.0323764 0.022196 2022-03-08T18:47:53.384 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:53.381678 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:53.628 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 183 16 131499 131483 44.8974 19.375 0.0065442 0.0222253 2022-03-08T18:47:54.627 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 184 16 131921 131905 44.7967 26.375 0.00694944 0.0223135 2022-03-08T18:47:55.627 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 185 16 132306 132290 44.6847 24.0625 0.0104675 0.0223675 2022-03-08T18:47:56.053 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:56.051339 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:56.627 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 186 16 132519 132503 44.516 13.3125 0.017406 0.0224559 2022-03-08T18:47:57.001 INFO:teuthology.orchestra.run.smithi002.stderr:+ '[' 11 '!=' 12 ']' 2022-03-08T18:47:57.002 INFO:teuthology.orchestra.run.smithi002.stderr:+ echo 'waiting for osd to come back up' 2022-03-08T18:47:57.002 INFO:teuthology.orchestra.run.smithi002.stdout:waiting for osd to come back up 2022-03-08T18:47:57.003 INFO:teuthology.orchestra.run.smithi002.stderr:++ toolbox ceph osd stat 2022-03-08T18:47:57.003 INFO:teuthology.orchestra.run.smithi002.stderr:++ kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd stat 2022-03-08T18:47:57.003 INFO:teuthology.orchestra.run.smithi002.stderr:++ cut -f3 -d ' ' 2022-03-08T18:47:57.098 INFO:teuthology.orchestra.run.smithi002.stderr:Unable to use a TTY - input is not a terminal or the right kind of file 2022-03-08T18:47:57.628 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 187 16 132971 132955 44.429 28.25 0.0135587 0.0225 2022-03-08T18:47:57.691 INFO:teuthology.orchestra.run.smithi002.stderr:+ num_osd=12 2022-03-08T18:47:57.691 INFO:teuthology.orchestra.run.smithi002.stderr:+ sleep 30s 2022-03-08T18:47:57.875 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:57.872651 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:47:58.628 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 188 16 133284 133268 44.2967 19.5625 0.00757168 0.0225594 2022-03-08T18:47:59.628 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 189 16 133676 133660 44.192 24.5 0.0771804 0.0226195 2022-03-08T18:47:59.667 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:47:59.661860 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:00.628 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 190 16 134133 134117 44.1097 28.5625 0.0607121 0.0226632 2022-03-08T18:48:01.435 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:01.433255 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:01.714 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 191 16 134374 134358 43.9576 15.0625 0.0134726 0.0226908 2022-03-08T18:48:02.628 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 192 16 134544 134528 43.784 10.625 0.0966626 0.0228296 2022-03-08T18:48:03.628 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 193 16 134799 134783 43.6397 15.9375 0.0118148 0.0228579 2022-03-08T18:48:04.629 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 194 16 135242 135226 43.5575 27.6875 0.0170058 0.0229497 2022-03-08T18:48:05.281 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:05.279291 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:05.629 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 195 16 135692 135676 43.4783 28.125 0.0152667 0.0229903 2022-03-08T18:48:06.629 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 196 16 135964 135948 43.3433 17 0.0140243 0.0230639 2022-03-08T18:48:06.908 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:06.906028 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:07.629 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 197 16 136349 136333 43.2454 24.0625 0.0199247 0.0231172 2022-03-08T18:48:08.629 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 198 16 136720 136704 43.1441 23.1875 0.011538 0.0231511 2022-03-08T18:48:08.669 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:08.667583 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:09.629 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 199 16 137018 137002 43.0209 18.625 0.0331902 0.0232358 2022-03-08T18:48:10.543 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:10.540814 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:10.629 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:48:10.627965+0000 min lat: 0.00170736 max lat: 15.2919 avg lat: 0.0232708 2022-03-08T18:48:10.630 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:48:10.630 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 200 16 137503 137487 42.9573 30.3125 0.00657988 0.0232708 2022-03-08T18:48:11.629 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 201 16 137833 137817 42.8462 20.625 0.0193565 0.0233203 2022-03-08T18:48:12.358 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:12.355515 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:12.629 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 202 16 138283 138267 42.7733 28.125 0.0203934 0.0233707 2022-03-08T18:48:13.629 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 203 16 138701 138685 42.6913 26.125 0.0140721 0.0234135 2022-03-08T18:48:14.039 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:14.037580 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:14.630 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 204 16 138717 138701 42.4869 1 0.0329301 0.0234142 2022-03-08T18:48:15.630 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 205 16 138756 138740 42.2916 2.4375 1.94725 0.0235455 2022-03-08T18:48:16.257 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:16.255148 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:16.630 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 206 16 139108 139092 42.1931 22 0.00586507 0.0236914 2022-03-08T18:48:17.630 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 207 16 139513 139497 42.1115 25.3125 0.0533592 0.0237372 2022-03-08T18:48:18.630 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 208 16 139956 139940 42.0422 27.6875 0.0290299 0.0237786 2022-03-08T18:48:18.851 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:18.849577 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:19.630 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 209 16 140381 140365 41.9681 26.5625 0.055524 0.0238203 2022-03-08T18:48:20.630 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 210 16 140792 140776 41.8906 25.6875 0.113659 0.0238605 2022-03-08T18:48:21.630 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 211 16 140874 140858 41.7163 5.125 0.0139445 0.0238831 2022-03-08T18:48:22.155 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:22.152712 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:22.630 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 212 16 141051 141035 41.5717 11.0625 0.0206269 0.0240415 2022-03-08T18:48:23.630 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 213 16 141471 141455 41.4998 26.25 0.0440826 0.0240865 2022-03-08T18:48:24.492 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:24.489886 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:24.631 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 214 16 141929 141913 41.4396 28.625 0.00794818 0.0241251 2022-03-08T18:48:25.631 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 215 16 142267 142251 41.3451 21.125 0.143392 0.0241769 2022-03-08T18:48:26.411 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:26.409101 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:26.631 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 216 16 142701 142685 41.2793 27.125 0.0171725 0.024217 2022-03-08T18:48:27.631 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 217 16 143015 142999 41.1795 19.625 0.0125605 0.0242741 2022-03-08T18:48:27.693 INFO:teuthology.orchestra.run.smithi002.stderr:+ '[' 12 '!=' 12 ']' 2022-03-08T18:48:27.695 INFO:teuthology.run_tasks:Running task rook.shell... 2022-03-08T18:48:27.710 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch status 2022-03-08T18:48:28.161 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:28.158841 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:28.631 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 218 16 143499 143483 41.1294 30.25 0.0140923 0.0243068 2022-03-08T18:48:29.631 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 219 16 143937 143921 41.0665 27.375 0.0394681 0.0243409 2022-03-08T18:48:30.419 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:30.416950 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:30.631 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:48:30.629979+0000 min lat: 0.00170736 max lat: 15.2919 avg lat: 0.0243426 2022-03-08T18:48:30.632 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:48:30.632 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 220 16 143987 143971 40.8941 3.125 0.00731019 0.0243426 2022-03-08T18:48:31.631 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 221 16 144390 144374 40.823 25.1875 0.023632 0.0244875 2022-03-08T18:48:32.374 INFO:teuthology.orchestra.run.smithi002.stdout:Backend: rook 2022-03-08T18:48:32.375 INFO:teuthology.orchestra.run.smithi002.stdout:Available: Yes 2022-03-08T18:48:32.399 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ps 2022-03-08T18:48:32.631 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 222 16 144725 144709 40.7334 20.9375 0.0263197 0.024543 2022-03-08T18:48:32.905 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:32.902391 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:32.905 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:32.902412 I | op-osd: waiting... 1 of 1 OSD prepare jobs have finished processing and 9 of 11 OSDs have been updated 2022-03-08T18:48:33.065 INFO:teuthology.orchestra.run.smithi002.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID 2022-03-08T18:48:33.066 INFO:teuthology.orchestra.run.smithi002.stdout:crashcollector.crash smithi002 running (7m) 0s ago 7m - - 8a96e9d18eed 2022-03-08T18:48:33.066 INFO:teuthology.orchestra.run.smithi002.stdout:crashcollector.crash smithi066 running (9m) 0s ago 9m - - 8a96e9d18eed 2022-03-08T18:48:33.067 INFO:teuthology.orchestra.run.smithi002.stdout:crashcollector.crash smithi137 running (9m) 0s ago 9m - - 8a96e9d18eed 2022-03-08T18:48:33.067 INFO:teuthology.orchestra.run.smithi002.stdout:mgr.a smithi002 running (9m) 0s ago 9m - - 8a96e9d18eed 2022-03-08T18:48:33.067 INFO:teuthology.orchestra.run.smithi002.stdout:mon.a smithi137 running (11m) 0s ago 11m - - 8a96e9d18eed 2022-03-08T18:48:33.067 INFO:teuthology.orchestra.run.smithi002.stdout:mon.b smithi066 running (10m) 0s ago 10m - - 8a96e9d18eed 2022-03-08T18:48:33.068 INFO:teuthology.orchestra.run.smithi002.stdout:mon.c smithi002 running (10m) 0s ago 10m - - 8a96e9d18eed 2022-03-08T18:48:33.068 INFO:teuthology.orchestra.run.smithi002.stdout:osd.0 smithi066 running (88s) 0s ago 88s - - 8a96e9d18eed 2022-03-08T18:48:33.069 INFO:teuthology.orchestra.run.smithi002.stdout:osd.1 smithi137 running (7m) 0s ago 7m - - 8a96e9d18eed 2022-03-08T18:48:33.069 INFO:teuthology.orchestra.run.smithi002.stdout:osd.10 smithi137 running (7m) 0s ago 7m - - 8a96e9d18eed 2022-03-08T18:48:33.069 INFO:teuthology.orchestra.run.smithi002.stdout:osd.11 smithi002 running (6m) 0s ago 6m - - 8a96e9d18eed 2022-03-08T18:48:33.070 INFO:teuthology.orchestra.run.smithi002.stdout:osd.2 smithi002 running (7m) 0s ago 7m - - 8a96e9d18eed 2022-03-08T18:48:33.070 INFO:teuthology.orchestra.run.smithi002.stdout:osd.3 smithi066 running (7m) 0s ago 7m - - 8a96e9d18eed 2022-03-08T18:48:33.070 INFO:teuthology.orchestra.run.smithi002.stdout:osd.4 smithi066 running (7m) 0s ago 7m - - 8a96e9d18eed 2022-03-08T18:48:33.070 INFO:teuthology.orchestra.run.smithi002.stdout:osd.5 smithi002 running (7m) 0s ago 7m - - 8a96e9d18eed 2022-03-08T18:48:33.071 INFO:teuthology.orchestra.run.smithi002.stdout:osd.6 smithi066 running (7m) 0s ago 7m - - 8a96e9d18eed 2022-03-08T18:48:33.071 INFO:teuthology.orchestra.run.smithi002.stdout:osd.7 smithi137 running (7m) 0s ago 7m - - 8a96e9d18eed 2022-03-08T18:48:33.071 INFO:teuthology.orchestra.run.smithi002.stdout:osd.8 smithi002 running (7m) 0s ago 7m - - 8a96e9d18eed 2022-03-08T18:48:33.071 INFO:teuthology.orchestra.run.smithi002.stdout:osd.9 smithi137 running (7m) 0s ago 7m - - 8a96e9d18eed 2022-03-08T18:48:33.091 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls 2022-03-08T18:48:33.632 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 223 16 145173 145157 40.6763 28 0.0675512 0.0245757 2022-03-08T18:48:34.049 INFO:teuthology.orchestra.run.smithi002.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2022-03-08T18:48:34.049 INFO:teuthology.orchestra.run.smithi002.stdout:crash 3/3 0s ago 9m * 2022-03-08T18:48:34.050 INFO:teuthology.orchestra.run.smithi002.stdout:mgr 1/1 0s ago 9m count:1 2022-03-08T18:48:34.050 INFO:teuthology.orchestra.run.smithi002.stdout:mon 3/3 0s ago 11m count:3 2022-03-08T18:48:34.050 INFO:teuthology.orchestra.run.smithi002.stdout:osd 24 0s ago 7m 2022-03-08T18:48:34.050 INFO:teuthology.orchestra.run.smithi002.stdout:osd.all-available-devices 0 0s ago - * 2022-03-08T18:48:34.076 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch host ls 2022-03-08T18:48:34.632 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 224 16 145603 145587 40.6147 26.875 0.0841788 0.0246126 2022-03-08T18:48:34.787 INFO:teuthology.orchestra.run.smithi002.stdout:HOST ADDR LABELS STATUS 2022-03-08T18:48:34.787 INFO:teuthology.orchestra.run.smithi002.stdout:smithi002 172.21.15.2/smithi002 2022-03-08T18:48:34.788 INFO:teuthology.orchestra.run.smithi002.stdout:smithi066 172.21.15.66/smithi066 2022-03-08T18:48:34.788 INFO:teuthology.orchestra.run.smithi002.stdout:smithi137 172.21.15.137/smithi137 2022-03-08T18:48:34.788 INFO:teuthology.orchestra.run.smithi002.stdout:3 hosts in cluster 2022-03-08T18:48:34.811 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch device ls 2022-03-08T18:48:35.360 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:35.358106 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:35.495 INFO:teuthology.orchestra.run.smithi002.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REFRESHED REJECT REASONS 2022-03-08T18:48:35.495 INFO:teuthology.orchestra.run.smithi002.stdout:smithi002 /dev/vg_nvme/lv_1 unknown None 107G 0s ago 2022-03-08T18:48:35.496 INFO:teuthology.orchestra.run.smithi002.stdout:smithi002 /dev/vg_nvme/lv_2 unknown None 107G 0s ago 2022-03-08T18:48:35.496 INFO:teuthology.orchestra.run.smithi002.stdout:smithi002 /dev/vg_nvme/lv_3 unknown None 107G 0s ago 2022-03-08T18:48:35.496 INFO:teuthology.orchestra.run.smithi002.stdout:smithi002 /dev/vg_nvme/lv_4 unknown None 107G 0s ago 2022-03-08T18:48:35.496 INFO:teuthology.orchestra.run.smithi002.stdout:smithi066 /dev/vg_nvme/lv_1 unknown None 107G 0s ago 2022-03-08T18:48:35.496 INFO:teuthology.orchestra.run.smithi002.stdout:smithi066 /dev/vg_nvme/lv_2 unknown None 107G 0s ago 2022-03-08T18:48:35.497 INFO:teuthology.orchestra.run.smithi002.stdout:smithi066 /dev/vg_nvme/lv_3 unknown None 107G 0s ago 2022-03-08T18:48:35.497 INFO:teuthology.orchestra.run.smithi002.stdout:smithi066 /dev/vg_nvme/lv_4 unknown None 107G 0s ago 2022-03-08T18:48:35.497 INFO:teuthology.orchestra.run.smithi002.stdout:smithi137 /dev/vg_nvme/lv_1 unknown None 107G 0s ago 2022-03-08T18:48:35.497 INFO:teuthology.orchestra.run.smithi002.stdout:smithi137 /dev/vg_nvme/lv_2 unknown None 107G 0s ago 2022-03-08T18:48:35.498 INFO:teuthology.orchestra.run.smithi002.stdout:smithi137 /dev/vg_nvme/lv_3 unknown None 107G 0s ago 2022-03-08T18:48:35.498 INFO:teuthology.orchestra.run.smithi002.stdout:smithi137 /dev/vg_nvme/lv_4 unknown None 107G 0s ago 2022-03-08T18:48:35.521 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch apply rgw foo 2022-03-08T18:48:35.632 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 225 16 146057 146041 40.5603 28.375 0.0547786 0.0246464 2022-03-08T18:48:36.610 INFO:teuthology.orchestra.run.smithi002.stdout:Created 2022-03-08T18:48:36.610 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:36.608190 I | ceph-spec: adding finalizer "cephobjectstore.ceph.rook.io" on "foo" 2022-03-08T18:48:36.632 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 226 16 146340 146324 40.459 17.6875 0.0717434 0.0247087 2022-03-08T18:48:36.634 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch apply mds foo 2022-03-08T18:48:37.309 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:37.307086 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:37.433 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:37.428780 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:48:37.633 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 227 16 146698 146682 40.3794 22.375 0.0264436 0.0247577 2022-03-08T18:48:38.063 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:38.060995 I | ceph-spec: adding finalizer "cephfilesystem.ceph.rook.io" on "foo" 2022-03-08T18:48:38.064 INFO:teuthology.orchestra.run.smithi002.stdout:Created 2022-03-08T18:48:38.089 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch apply rbd-mirror 2022-03-08T18:48:38.632 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 228 16 146879 146863 40.2519 11.3125 0.0160502 0.0247718 2022-03-08T18:48:38.977 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:38.972024 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:48:39.342 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:39.339450 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:39.430 INFO:teuthology.orchestra.run.smithi002.stdout:Success 2022-03-08T18:48:39.454 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:39.451926 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:48:39.463 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch apply nfs foo --port 12777 2022-03-08T18:48:39.632 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 229 16 146978 146962 40.1031 6.1875 0.0115901 0.0249206 2022-03-08T18:48:40.185 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:40.182046 W | ceph-rbd-mirror-controller: (DEPRECATED) use of peer secret names in CephRBDMirror is deprecated. Please use CephBlockPool CR to configure peer secret names and import peers. 2022-03-08T18:48:40.187 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:40.182079 I | ceph-rbd-mirror-controller: configure rbd-mirroring with 1 workers 2022-03-08T18:48:40.187 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:40.182092 I | cephclient: getting or creating ceph auth key "client.rbd-mirror.a" 2022-03-08T18:48:40.632 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 230 16 147356 147340 40.0315 23.625 0.0521859 0.0249736 2022-03-08T18:48:41.522 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:41.520176 I | ceph-rbd-mirror-controller: "rook-ceph-rbd-mirror-a" deployment started 2022-03-08T18:48:41.536 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:41.528046 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:48:41.632 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:41.629613 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:48:41.633 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:41.629740 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:48:41.633 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 231 16 147793 147777 39.9764 27.3125 0.0199459 0.0250066 2022-03-08T18:48:42.029 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:42.026175 E | ceph-crashcollector-controller: node reconcile failed on op "unchanged": Operation cannot be fulfilled on deployments.apps "rook-ceph-crashcollector-smithi066": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T18:48:42.145 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:42.137315 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:48:42.633 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 232 16 148186 148170 39.91 24.5625 0.0080668 0.025046 2022-03-08T18:48:42.831 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:42.829172 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:43.041 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:43.039143 I | ceph-file-controller: creating filesystem "foo" 2022-03-08T18:48:43.633 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 233 16 148471 148455 39.8151 17.8125 0.0633701 0.0251026 2022-03-08T18:48:44.480 INFO:teuthology.orchestra.run.smithi002.stdout:Unable to create NFS daemon, check logs for more traceback 2022-03-08T18:48:44.481 INFO:teuthology.orchestra.run.smithi002.stdout:'RookOrchestrator' object has no attribute 'open_ioctx' 2022-03-08T18:48:44.537 DEBUG:teuthology.run_tasks:Unwinding manager radosbench 2022-03-08T18:48:44.550 INFO:tasks.radosbench:joining radosbench (timing out after 11100s) 2022-03-08T18:48:44.551 INFO:teuthology.orchestra.run:waiting for 11100 2022-03-08T18:48:44.633 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 234 16 148813 148797 39.7363 21.375 0.111991 0.0251574 2022-03-08T18:48:45.633 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 235 16 149243 149227 39.6816 26.875 0.0148185 0.0251914 2022-03-08T18:48:46.235 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:46.233145 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:46.633 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 236 16 149683 149667 39.63 27.5 0.0145151 0.0252273 2022-03-08T18:48:47.633 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 237 16 150088 150072 39.5695 25.3125 0.0849035 0.0252649 2022-03-08T18:48:48.633 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 238 16 150456 150440 39.4999 23 0.110893 0.0253094 2022-03-08T18:48:49.634 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 239 16 150868 150852 39.4424 25.75 0.0154864 0.0253338 2022-03-08T18:48:50.634 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:48:50.632276+0000 min lat: 0.00170736 max lat: 15.2919 avg lat: 0.025394 2022-03-08T18:48:50.634 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:48:50.634 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 240 16 151209 151193 39.3668 21.3125 0.00992983 0.025394 2022-03-08T18:48:51.449 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:51.447317 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:51.634 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 241 16 151654 151638 39.3189 27.8125 0.0258908 0.0254248 2022-03-08T18:48:51.867 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:51.862063 I | cephclient: setting pool property "compression_mode" to "none" on pool "foo-metadata" 2022-03-08T18:48:52.634 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 242 16 152092 152076 39.2695 27.375 0.00881165 0.0254577 2022-03-08T18:48:53.634 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 243 16 152482 152466 39.2082 24.375 0.00889555 0.025497 2022-03-08T18:48:53.675 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:53.672412 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:53.890 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:53.883224 I | cephclient: setting pool property "compression_mode" to "none" on pool "foo.rgw.buckets.index" 2022-03-08T18:48:53.894 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:53.892147 I | cephclient: creating replicated pool foo-metadata succeeded 2022-03-08T18:48:53.895 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:53.892212 I | cephclient: setting pool property "compression_mode" to "none" on pool "foo.rgw.buckets.non-ec" 2022-03-08T18:48:53.909 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:53.902940 I | cephclient: setting pool property "compression_mode" to "none" on pool "foo.rgw.control" 2022-03-08T18:48:54.001 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:53.995563 I | cephclient: setting pool property "compression_mode" to "none" on pool "foo.rgw.meta" 2022-03-08T18:48:54.041 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:54.035555 I | cephclient: setting pool property "compression_mode" to "none" on pool "foo.rgw.log" 2022-03-08T18:48:54.634 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 244 16 152896 152880 39.1536 25.875 0.0952734 0.0255317 2022-03-08T18:48:55.635 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 245 16 153264 153248 39.0876 23 0.0534376 0.0255748 2022-03-08T18:48:56.634 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 246 16 153479 153463 38.9834 13.4375 0.0111901 0.0255983 2022-03-08T18:48:57.634 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 247 16 153485 153469 38.827 0.375 1.49202 0.0256543 2022-03-08T18:48:58.558 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:48:58.556361 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:48:58.634 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 248 16 153908 153892 38.7771 26.4375 0.018627 0.0257788 2022-03-08T18:48:59.635 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 249 16 154293 154277 38.718 24.0625 0.054579 0.0258191 2022-03-08T18:49:00.013 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:00.010549 I | cephclient: creating replicated pool foo.rgw.meta succeeded 2022-03-08T18:49:00.016 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:00.012818 I | cephclient: creating replicated pool foo.rgw.control succeeded 2022-03-08T18:49:00.635 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 250 16 154692 154676 38.6629 24.9375 0.0158402 0.0258543 2022-03-08T18:49:01.248 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:01.246326 I | cephclient: creating replicated pool foo.rgw.log succeeded 2022-03-08T18:49:01.635 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 251 16 155146 155130 38.6219 28.375 0.0120452 0.0258844 2022-03-08T18:49:01.950 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:01.947415 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:02.635 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 252 16 155460 155444 38.5465 19.625 0.0163826 0.0259294 2022-03-08T18:49:03.636 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 253 16 155870 155854 38.4954 25.625 0.0285117 0.0259672 2022-03-08T18:49:03.939 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:03.936821 I | cephclient: setting pool property "compression_mode" to "none" on pool "foo-data0" 2022-03-08T18:49:04.282 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:04.279636 I | cephclient: creating replicated pool foo.rgw.buckets.non-ec succeeded 2022-03-08T18:49:04.292 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:04.288816 I | cephclient: creating replicated pool foo.rgw.buckets.index succeeded 2022-03-08T18:49:04.363 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:04.360964 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:49:04.363 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:04.360985 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:49:04.382 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:04.380507 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:49:04.635 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 254 16 156133 156117 38.4085 16.4375 0.0306909 0.0260277 2022-03-08T18:49:04.917 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:04.915300 I | cephclient: creating replicated pool foo-data0 succeeded 2022-03-08T18:49:04.918 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:04.915335 I | cephclient: creating filesystem "foo" with metadata pool "foo-metadata" and data pools [foo-data0] 2022-03-08T18:49:05.250 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:05.246383 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:05.635 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 255 16 156572 156556 38.3655 27.4375 0.0271437 0.026058 2022-03-08T18:49:06.295 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:06.292893 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:49:06.461 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:06.457888 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:49:06.462 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:06.457926 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:49:06.635 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 256 16 156998 156982 38.3196 26.625 0.0114999 0.0260891 2022-03-08T18:49:07.083 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:07.079908 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:49:07.109 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:07.095084 I | ceph-file-controller: created filesystem "foo" on 1 data pool(s) and metadata pool "foo-metadata" 2022-03-08T18:49:07.605 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:07.536732 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:07.635 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 257 16 157344 157328 38.2547 21.625 0.076376 0.0261321 2022-03-08T18:49:08.636 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 258 16 157410 157394 38.1224 4.125 0.050297 0.0261361 2022-03-08T18:49:09.555 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:09.552217 I | cephclient: setting allow_standby_replay for filesystem "foo" 2022-03-08T18:49:09.636 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 259 16 157507 157491 37.9986 6.0625 0.00768438 0.026278 2022-03-08T18:49:10.636 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:49:10.634496+0000 min lat: 0.00170736 max lat: 15.2919 avg lat: 0.0263561 2022-03-08T18:49:10.636 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:49:10.637 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 260 16 157836 157820 37.9315 20.5625 0.0171576 0.0263561 2022-03-08T18:49:11.281 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:11.279136 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:49:11.483 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:11.479522 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:11.511 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:11.508226 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:49:11.637 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 261 16 158182 158166 37.869 21.625 0.0503044 0.0263963 2022-03-08T18:49:11.989 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:11.981684 I | ceph-file-controller: start running mdses for filesystem "foo" 2022-03-08T18:49:12.613 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:12.611331 I | cephclient: getting or creating ceph auth key "mds.foo-a" 2022-03-08T18:49:12.636 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 262 16 158628 158612 37.8309 27.875 0.00858658 0.0264248 2022-03-08T18:49:13.333 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:13.329940 I | op-mds: setting mds config flags 2022-03-08T18:49:13.333 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:13.329971 I | op-config: setting "mds.foo-a"="mds_join_fs"="foo" option to the mon configuration database 2022-03-08T18:49:13.451 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:13.431168 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:49:13.561 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:13.557310 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:49:13.562 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:13.557330 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:49:13.608 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:13.606440 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:13.636 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 263 16 159026 159010 37.7816 24.875 0.0118584 0.0264595 2022-03-08T18:49:14.028 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:14.025690 I | op-config: successfully set "mds.foo-a"="mds_join_fs"="foo" option to the mon configuration database 2022-03-08T18:49:14.031 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:14.029613 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:49:14.036 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:14.034384 I | cephclient: getting or creating ceph auth key "mds.foo-b" 2022-03-08T18:49:14.637 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 264 16 159381 159365 37.7226 22.1875 0.041535 0.0265012 2022-03-08T18:49:15.637 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 265 16 159553 159537 37.6208 10.75 0.0105198 0.0265143 2022-03-08T18:49:16.076 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:16.034325 I | op-mds: setting mds config flags 2022-03-08T18:49:16.077 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:16.034343 I | op-config: setting "mds.foo-b"="mds_join_fs"="foo" option to the mon configuration database 2022-03-08T18:49:16.514 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:16.511867 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:16.637 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 266 16 159567 159551 37.4826 0.875 1.60446 0.0266192 2022-03-08T18:49:17.091 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:17.089429 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:49:17.116 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:17.114234 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:49:17.637 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 267 16 160000 159984 37.4436 27.0625 0.0485126 0.0266997 2022-03-08T18:49:17.927 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:17.924545 I | op-config: successfully set "mds.foo-b"="mds_join_fs"="foo" option to the mon configuration database 2022-03-08T18:49:18.637 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 268 16 160444 160428 37.4074 27.75 0.0593782 0.0267256 2022-03-08T18:49:19.384 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:19.381356 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:49:19.457 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:19.454934 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:49:19.457 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:19.454972 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:49:19.611 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:19.609321 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:19.637 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 269 16 160609 160593 37.3067 10.3125 0.00723053 0.0267328 2022-03-08T18:49:19.839 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:19.836694 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:49:20.637 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 270 16 160717 160701 37.1935 6.75 0.0737454 0.0268784 2022-03-08T18:49:21.637 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 271 16 161107 161091 37.1462 24.375 0.00904295 0.0269118 2022-03-08T18:49:22.108 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:22.105699 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:22.637 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 272 16 161530 161514 37.1068 26.4375 0.0160079 0.0269405 2022-03-08T18:49:23.637 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 273 16 161933 161917 37.0632 25.1875 0.0430506 0.0269734 2022-03-08T18:49:23.912 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:23.910110 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:49:23.958 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:23.955646 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:49:24.638 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 274 16 162313 162297 37.0146 23.75 0.075127 0.0270089 2022-03-08T18:49:24.881 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:24.878605 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:25.567 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:25.564740 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:49:25.638 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 275 16 162737 162721 36.9763 26.5 0.0577698 0.0270367 2022-03-08T18:49:25.714 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:25.711662 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:49:25.714 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:25.711683 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:49:26.133 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:26.131289 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:49:26.638 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 276 16 163089 163073 36.9221 22 0.0160252 0.0270772 2022-03-08T18:49:27.260 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:27.258087 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:27.638 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 277 16 163294 163278 36.835 12.8125 0.0195186 0.027141 2022-03-08T18:49:28.638 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 278 16 163718 163702 36.7978 26.5 0.0340516 0.0271688 2022-03-08T18:49:29.638 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 279 16 164151 164135 36.7629 27.0625 0.0211902 0.0271931 2022-03-08T18:49:30.160 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:30.132185 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:30.638 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:49:30.636563+0000 min lat: 0.00170736 max lat: 15.2919 avg lat: 0.0272211 2022-03-08T18:49:30.639 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:49:30.639 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 280 16 164572 164556 36.7256 26.3125 0.0426818 0.0272211 2022-03-08T18:49:31.001 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:30.999036 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:49:31.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:31.084461 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:49:31.638 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 281 16 165025 165009 36.6957 28.3125 0.0562254 0.0272438 2022-03-08T18:49:32.638 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 282 16 165451 165435 36.6599 26.625 0.0107653 0.0272703 2022-03-08T18:49:33.489 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:33.486089 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:49:33.639 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 283 16 165788 165772 36.6048 21.0625 0.00754726 0.0272965 2022-03-08T18:49:34.056 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:34.052486 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:34.057 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:34.052508 I | op-osd: waiting... 1 of 1 OSD prepare jobs have finished processing and 9 of 11 OSDs have been updated 2022-03-08T18:49:34.639 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 284 16 165790 165774 36.4764 0.125 0.0266307 0.0272964 2022-03-08T18:49:35.639 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 285 16 166106 166090 36.4177 19.75 0.0834883 0.0274489 2022-03-08T18:49:36.639 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 286 16 166477 166461 36.3714 23.1875 0.0992958 0.0274856 2022-03-08T18:49:36.778 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:36.775553 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:49:36.778 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:36.775579 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:49:37.173 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:37.165946 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:49:37.639 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 287 16 166845 166829 36.3248 23 0.0373577 0.0275206 2022-03-08T18:49:38.052 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:38.050263 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:38.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 288 16 167212 167196 36.2783 22.9375 0.0435444 0.027557 2022-03-08T18:49:39.639 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 289 16 167528 167512 36.2211 19.75 0.0733141 0.0275998 2022-03-08T18:49:40.639 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 290 16 168046 168030 36.2079 32.375 0.077669 0.027611 2022-03-08T18:49:40.965 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:40.962281 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:41.639 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 291 16 168453 168437 36.1709 25.4375 0.00754774 0.0276351 2022-03-08T18:49:42.639 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 292 16 168677 168661 36.0949 14 0.0165348 0.0276965 2022-03-08T18:49:42.818 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:42.815646 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:49:42.983 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:42.980496 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:49:43.086 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:43.083888 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:43.639 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 293 16 168939 168923 36.0276 16.375 0.109502 0.0277473 2022-03-08T18:49:44.640 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 294 15 169382 169367 35.9995 27.75 0.016914 0.0277713 2022-03-08T18:49:44.920 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:44.917461 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:49:44.993 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:44.991011 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:49:44.994 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:44.991046 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:49:45.460 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:45.456143 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:49:45.640 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 295 16 169814 169798 35.9687 26.9375 0.120086 0.0277898 2022-03-08T18:49:45.872 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:45.869110 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:46.640 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 296 16 170178 170162 35.9241 22.75 0.0191011 0.0278157 2022-03-08T18:49:47.640 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 297 16 170316 170300 35.8322 8.625 0.0278573 0.0279007 2022-03-08T18:49:48.152 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:48.147232 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:48.359 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:48.356547 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:49:48.640 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 298 16 170842 170826 35.8222 32.875 0.016644 0.0279085 2022-03-08T18:49:48.683 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:48.680858 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:49:49.640 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 299 16 171341 171325 35.8067 31.1875 0.0195942 0.0279204 2022-03-08T18:49:50.283 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:50.280994 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:50.321 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:50.318603 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:49:50.477 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:50.472308 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:49:50.478 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:50.472335 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:49:50.640 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:49:50.638502+0000 min lat: 0.00170736 max lat: 15.2919 avg lat: 0.0279328 2022-03-08T18:49:50.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:49:50.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 300 16 171758 171742 35.7742 26.0625 0.0104371 0.0279328 2022-03-08T18:49:51.272 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:51.257815 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:49:51.640 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 301 16 172092 172076 35.7247 20.875 0.0123928 0.0279853 2022-03-08T18:49:52.366 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:52.363413 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:52.640 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 302 16 172353 172337 35.6604 16.3125 0.0100888 0.0280168 2022-03-08T18:49:53.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 303 16 172800 172784 35.6349 27.9375 0.0388368 0.0280534 2022-03-08T18:49:53.825 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:53.822102 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:49:54.376 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:54.374275 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:54.469 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:54.467036 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:49:54.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 304 16 173322 173306 35.625 32.625 0.00622581 0.0280622 2022-03-08T18:49:55.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 305 16 173883 173867 35.6232 35.0625 0.0371194 0.028064 2022-03-08T18:49:56.543 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:56.540796 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:49:56.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 306 16 174410 174394 35.6144 32.9375 0.00852405 0.0280716 2022-03-08T18:49:56.726 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:56.724010 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:49:56.727 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:56.724029 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:49:56.783 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:56.780632 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:49:57.185 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:57.182857 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:49:57.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 307 16 174931 174915 35.6044 32.5625 0.00846893 0.0280773 2022-03-08T18:49:58.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 308 16 175038 175022 35.5106 6.6875 0.00410321 0.028109 2022-03-08T18:49:59.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 309 16 175584 175568 35.5061 34.125 0.0143027 0.028157 2022-03-08T18:49:59.921 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:49:59.918227 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:00.332 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:00.329998 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:50:00.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 310 16 176120 176104 35.4996 33.5 0.0119636 0.0281627 2022-03-08T18:50:01.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 311 16 176724 176708 35.5068 37.75 0.0201845 0.0281556 2022-03-08T18:50:01.763 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:01.761270 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:01.814 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:01.812069 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:50:02.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 312 16 177274 177258 35.5032 34.375 0.0424246 0.0281593 2022-03-08T18:50:03.291 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:03.288430 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:50:03.424 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:03.421950 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:50:03.425 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:03.421968 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:50:03.639 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:03.634796 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:03.641 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 313 16 177636 177620 35.462 22.625 0.00485888 0.0281841 2022-03-08T18:50:04.642 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 314 16 177944 177928 35.4104 19.25 0.0124547 0.0282333 2022-03-08T18:50:05.642 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 315 16 178426 178410 35.3936 30.125 0.0954931 0.0282419 2022-03-08T18:50:05.987 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:05.984354 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:50:06.642 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 316 16 178488 178472 35.2938 3.875 0.00463985 0.0282478 2022-03-08T18:50:07.631 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:07.629017 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:07.642 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 317 16 178793 178777 35.2426 19.0625 0.03568 0.0283645 2022-03-08T18:50:08.201 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:08.198879 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:50:08.642 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 318 16 179306 179290 35.2326 32.0625 0.00550894 0.0283759 2022-03-08T18:50:09.522 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:09.519395 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:09.642 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 319 16 179655 179639 35.1906 21.8125 0.00395913 0.02841 2022-03-08T18:50:10.643 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:50:10.640585+0000 min lat: 0.00170736 max lat: 15.2919 avg lat: 0.0284356 2022-03-08T18:50:10.643 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:50:10.643 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 320 16 180011 179995 35.1501 22.25 0.0155917 0.0284356 2022-03-08T18:50:10.773 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:10.769609 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:50:11.351 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:11.349128 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:11.643 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 321 16 180586 180570 35.1526 35.9375 0.0409667 0.0284404 2022-03-08T18:50:12.166 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:12.163213 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:50:12.486 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:12.483530 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:50:12.498 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:12.483563 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:50:12.643 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 322 16 181064 181048 35.1362 29.875 0.0160925 0.028453 2022-03-08T18:50:12.933 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:12.930752 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:50:13.643 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 323 16 181495 181479 35.1108 26.9375 0.0041944 0.0284649 2022-03-08T18:50:14.495 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:14.492535 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:14.643 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 324 16 181890 181874 35.0786 24.6875 0.00430898 0.0284858 2022-03-08T18:50:15.643 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 325 16 181949 181933 34.982 3.6875 0.00443074 0.0285253 2022-03-08T18:50:16.643 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 326 16 181987 181971 34.882 2.375 0.00809615 0.0286599 2022-03-08T18:50:16.704 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:16.701543 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:50:16.747 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:16.745099 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:17.644 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 327 16 182454 182438 34.8646 29.1875 0.0628959 0.0286756 2022-03-08T18:50:18.643 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 328 16 182932 182916 34.8493 29.875 0.0515788 0.0286884 2022-03-08T18:50:19.500 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:19.497552 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:19.643 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 329 16 183446 183430 34.841 32.125 0.0655723 0.0286938 2022-03-08T18:50:20.644 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 330 16 183981 183965 34.8368 33.4375 0.0432307 0.0286975 2022-03-08T18:50:21.644 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 331 16 184500 184484 34.8295 32.4375 0.0142029 0.0287029 2022-03-08T18:50:21.831 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:21.828422 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:50:22.644 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 332 16 184806 184790 34.7822 19.125 0.0114614 0.0287429 2022-03-08T18:50:22.713 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:22.711410 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:23.231 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:23.228684 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:50:23.370 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:23.368044 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:50:23.371 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:23.368080 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:50:23.644 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 333 16 185086 185070 34.7303 17.5 0.0206178 0.028785 2022-03-08T18:50:23.847 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:23.845169 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:50:24.644 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 334 16 185533 185517 34.71 27.9375 0.0189154 0.0288031 2022-03-08T18:50:24.989 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:24.986526 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:25.644 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 335 16 186082 186066 34.7088 34.3125 0.00731502 0.0288044 2022-03-08T18:50:26.334 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:26.331611 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:50:26.644 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 336 16 186617 186601 34.705 33.4375 0.00878762 0.0288072 2022-03-08T18:50:26.865 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:26.860933 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:27.644 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 337 16 187166 187150 34.7038 34.3125 0.0892326 0.0288072 2022-03-08T18:50:28.630 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:28.628039 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:28.644 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 338 16 187475 187459 34.6583 19.3125 0.0121036 0.0288361 2022-03-08T18:50:29.645 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 339 16 187913 187897 34.6368 27.375 0.0179917 0.0288634 2022-03-08T18:50:30.482 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:30.479608 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:30.645 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:50:30.642645+0000 min lat: 0.00170736 max lat: 15.2919 avg lat: 0.0288653 2022-03-08T18:50:30.645 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:50:30.646 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 340 16 188452 188436 34.634 33.6875 0.00882033 0.0288653 2022-03-08T18:50:31.645 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 341 16 188750 188734 34.587 18.625 0.0119932 0.0289053 2022-03-08T18:50:32.208 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:32.205852 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:32.286 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:32.283154 I | op-osd: waiting... 1 of 1 OSD prepare jobs have finished processing and 9 of 11 OSDs have been updated 2022-03-08T18:50:32.645 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 342 16 189286 189270 34.5838 33.5 0.0703166 0.0289076 2022-03-08T18:50:33.645 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 343 16 189790 189774 34.5748 31.5 0.0140163 0.0289134 2022-03-08T18:50:34.058 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:34.056195 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:34.645 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 344 16 190039 190023 34.5196 15.5625 0.0930407 0.0289622 2022-03-08T18:50:35.645 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 345 16 190607 190591 34.5224 35.5 0.0571662 0.028959 2022-03-08T18:50:35.868 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:35.865943 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:36.579 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:36.576727 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:50:36.645 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 346 16 191152 191136 34.5211 34.0625 0.027425 0.0289607 2022-03-08T18:50:37.645 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 347 16 191662 191646 34.5134 31.875 0.0133001 0.028967 2022-03-08T18:50:37.941 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:37.938933 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:37.991 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:37.986095 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:50:38.144 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:38.141621 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:50:38.144 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:38.141642 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:50:38.628 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:38.626045 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:50:38.646 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 348 16 192195 192179 34.51 33.3125 0.00768289 0.0289684 2022-03-08T18:50:39.646 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 349 16 192312 192296 34.432 7.3125 0.0694375 0.0290214 2022-03-08T18:50:40.567 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:40.564468 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:40.646 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 350 16 192535 192519 34.3735 13.9375 0.0148507 0.0290531 2022-03-08T18:50:41.366 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:41.363709 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:50:41.646 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 351 16 192741 192725 34.3122 12.875 0.0390619 0.029137 2022-03-08T18:50:42.646 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 352 16 193233 193217 34.3021 30.75 0.141402 0.0291446 2022-03-08T18:50:43.505 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:43.502247 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:43.647 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 353 16 193739 193723 34.2945 31.625 0.0637295 0.0291504 2022-03-08T18:50:44.646 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 354 16 194158 194142 34.2716 26.1875 0.0219044 0.0291554 2022-03-08T18:50:45.509 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:45.506824 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:45.646 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 355 16 194601 194585 34.253 27.6875 0.00938816 0.0291873 2022-03-08T18:50:46.646 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 356 16 195060 195044 34.2374 28.6875 0.0450711 0.0292011 2022-03-08T18:50:47.291 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:47.287655 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:47.646 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 357 16 195540 195524 34.2255 30 0.0194908 0.0292067 2022-03-08T18:50:48.647 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 358 16 195927 195911 34.1975 24.1875 0.00817706 0.0292357 2022-03-08T18:50:49.423 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:49.417813 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:49.647 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 359 16 196473 196457 34.1973 34.125 0.00878008 0.0292356 2022-03-08T18:50:50.648 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:2022-03-08T18:50:50.644727+0000 min lat: 0.00170736 max lat: 15.2919 avg lat: 0.0292356 2022-03-08T18:50:50.648 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2022-03-08T18:50:50.649 INFO:tasks.radosbench.radosbench.a.smithi002.stdout: 360 11 196959 196948 34.1875 30.6875 0.0738158 0.0292356 2022-03-08T18:50:51.158 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:51.156178 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:51.648 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Total time run: 360.42 2022-03-08T18:50:51.649 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Total writes made: 196959 2022-03-08T18:50:51.649 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Write size: 65536 2022-03-08T18:50:51.649 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Object size: 65536 2022-03-08T18:50:51.649 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Bandwidth (MB/sec): 34.1545 2022-03-08T18:50:51.650 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Stddev Bandwidth: 28.3604 2022-03-08T18:50:51.650 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Max bandwidth (MB/sec): 173.938 2022-03-08T18:50:51.650 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Min bandwidth (MB/sec): 0 2022-03-08T18:50:51.650 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Average IOPS: 546 2022-03-08T18:50:51.651 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Stddev IOPS: 453.774 2022-03-08T18:50:51.651 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Max IOPS: 2783 2022-03-08T18:50:51.651 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Min IOPS: 0 2022-03-08T18:50:51.651 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Average Latency(s): 0.029262 2022-03-08T18:50:51.652 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Stddev Latency(s): 0.212622 2022-03-08T18:50:51.652 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Max latency(s): 15.2919 2022-03-08T18:50:51.652 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Min latency(s): 0.00170736 2022-03-08T18:50:51.653 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Cleaning up (deleting benchmark objects) 2022-03-08T18:50:55.007 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:55.004739 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:50:58.707 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:50:58.705204 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:00.881 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:00.879285 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:01.966 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:01.951980 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:51:02.608 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:02.606298 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:03.433 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:03.430209 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:51:03.576 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:03.571551 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:51:03.576 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:03.571577 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:51:04.003 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:04.000649 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:51:04.931 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:04.927798 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:06.297 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:06.294772 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:51:06.876 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:06.873742 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:08.592 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:08.589722 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:10.654 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:10.651725 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:12.479 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:12.474498 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:14.071 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:14.069397 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:15.948 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:15.945636 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:17.688 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:17.685900 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:19.472 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:19.470351 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:21.445 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:21.441191 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:23.605 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:23.602994 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:25.349 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:25.346004 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:27.128 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:27.126389 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:28.968 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:28.965833 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:32.796 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:32.793933 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:32.798 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:32.794375 I | op-osd: waiting... 1 of 1 OSD prepare jobs have finished processing and 9 of 11 OSDs have been updated 2022-03-08T18:51:36.146 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:36.143687 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:37.987 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:37.985168 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:39.795 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:39.792669 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:41.622 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:41.620044 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:43.439 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:43.436573 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:45.220 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:45.218078 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:47.036 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:47.033416 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:47.271 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:47.267656 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:51:48.706 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:48.703387 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:51:48.811 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:48.809249 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:48.875 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:48.872657 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:51:48.875 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:48.872681 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:51:49.356 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:49.352533 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:51:51.082 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:51.080184 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:51.567 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:51.565494 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:51:53.508 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:53.505857 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:55.307 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:55.304730 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:56.854 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:56.845209 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:51:58.850 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:51:58.848348 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:00.629 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:00.627534 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:02.650 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:02.648164 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:04.270 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:04.267949 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:05.931 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:05.929151 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:07.682 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:07.666232 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:09.619 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:09.566057 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:12.580 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:12.577249 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:14.282 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:14.279661 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:16.345 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:16.342748 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:18.128 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:18.125673 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:19.905 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:19.903282 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:21.768 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:21.765770 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:23.583 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:23.581390 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:25.383 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:25.381567 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:27.396 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:27.391912 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:29.005 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:28.998306 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:30.795 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:30.793378 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:32.385 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:32.382537 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:32.385 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:32.382595 I | op-osd: waiting... 1 of 1 OSD prepare jobs have finished processing and 9 of 11 OSDs have been updated 2022-03-08T18:52:34.103 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:34.101688 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:35.914 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:35.911595 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:37.521 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:37.518227 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:39.473 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:39.471198 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:41.209 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:41.206757 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:42.996 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:42.994555 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:44.706 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:44.703846 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:46.570 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:46.568048 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:48.274 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:48.271985 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:50.362 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:50.359824 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:51.950 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:51.948549 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:53.616 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:53.614284 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:55.483 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:55.480849 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:57.699 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:57.685991 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:52:59.500 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:52:59.497897 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:01.243 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:01.240938 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:03.515 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:03.513520 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:05.239 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:05.237286 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:06.949 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:06.945392 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:08.561 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:08.559297 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:10.286 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:10.283795 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:11.948 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:11.946307 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:13.493 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:13.491267 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:53:13.578 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:13.576035 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:14.942 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:14.940706 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:53:15.017 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:15.015457 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:53:15.017 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:15.015481 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:53:15.445 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:15.442950 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:15.457 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:15.455349 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:53:17.544 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:17.539558 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:18.138 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:18.135953 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:53:19.575 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:19.573779 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:21.338 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:21.335421 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:23.098 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:23.095951 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:24.784 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:24.782048 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:26.800 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:26.797727 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:28.501 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:28.486369 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:30.285 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:30.283657 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:32.312 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:32.310826 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:32.313 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:32.310863 I | op-osd: waiting... 1 of 1 OSD prepare jobs have finished processing and 9 of 11 OSDs have been updated 2022-03-08T18:53:34.045 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:34.040750 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:35.748 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:35.745967 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:37.477 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:37.475914 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:39.170 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:39.167568 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:40.763 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:40.761180 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:42.222 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:42.220269 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:43.904 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:43.902218 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:46.498 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:46.496676 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:48.121 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:48.119119 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:49.911 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:49.909990 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:51.681 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:51.679374 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:53.732 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:53.729364 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:55.469 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:55.467077 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:57.236 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:57.234090 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:53:59.226 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:53:59.224405 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:01.003 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:01.001554 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:02.650 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:02.648658 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:04.281 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:04.279646 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:06.050 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:06.048114 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:07.791 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:07.789890 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:09.504 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:09.502747 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:11.145 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:11.144054 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:12.854 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:12.853172 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:14.711 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:14.709321 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:16.384 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:16.382674 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:18.096 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:18.095370 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:19.901 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:19.899435 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:21.647 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:21.645895 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:23.664 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:23.663224 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:26.379 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:26.378447 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:28.178 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:28.176976 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:29.894 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:29.892835 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:31.657 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:31.655956 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:33.499 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:33.496950 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:33.505 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:33.496997 I | op-osd: waiting... 1 of 1 OSD prepare jobs have finished processing and 9 of 11 OSDs have been updated 2022-03-08T18:54:35.664 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:35.661974 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:37.348 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:37.337525 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:39.051 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:39.050290 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:40.779 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:40.772460 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:42.546 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:42.545087 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:44.215 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:44.213546 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:45.836 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:45.834475 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:47.558 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:47.557306 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:49.698 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:49.697419 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:51.459 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:51.458208 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:53.184 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:53.182836 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:54.764 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:54.763048 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:56.510 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:56.509193 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:58.150 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:58.148845 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:54:59.864 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:54:59.862792 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:01.561 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:01.560335 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:05.417 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:05.416501 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:07.159 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:07.158612 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:08.791 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:08.789811 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:10.541 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:10.540522 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:12.929 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:12.928308 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:14.589 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:14.588124 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:16.525 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:16.523615 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:18.426 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:18.424540 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:20.166 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:20.165347 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:21.865 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:21.863341 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:23.623 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:23.622177 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:25.470 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:25.469177 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:27.233 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:27.232409 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:28.880 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:28.879783 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:30.716 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:30.715071 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:32.548 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:32.547347 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:32.548 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:32.547392 I | op-osd: waiting... 1 of 1 OSD prepare jobs have finished processing and 9 of 11 OSDs have been updated 2022-03-08T18:55:34.216 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:34.215276 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:35.935 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:35.934305 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:37.670 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:37.669401 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:39.289 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:39.287561 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:41.043 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:41.042183 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:44.824 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:44.822906 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:46.532 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:46.530993 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:48.308 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:48.307806 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:50.419 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:50.418404 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:52.217 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:52.216960 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:53.941 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:53.940769 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:56.074 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:56.072638 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:57.814 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:57.813256 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:55:59.584 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:55:59.583288 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:01.199 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:01.198082 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:01.986 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:01.985295 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:56:03.015 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:03.014419 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:03.784 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:03.783552 I | ceph-object-controller: reconciling object store deployments 2022-03-08T18:56:03.852 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:03.851541 I | ceph-object-controller: ceph object store gateway service running at 10.106.126.195 2022-03-08T18:56:03.852 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:03.851571 I | ceph-object-controller: reconciling object store pools 2022-03-08T18:56:04.347 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:04.346050 W | ceph-object-controller: failed to adjust the PG count for rgw metadata pools. using the general default. failed to get config setting "rgw_rados_pool_pg_num_min" for user "mon.": exit status 2 2022-03-08T18:56:05.167 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:05.166236 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:06.586 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:06.585704 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to create object store deployments: failed to create object pools: failed to create metadata pools: failed to create pool .rgw.root for object store foo.: failed to set size property to replicated pool ".rgw.root" to 3: failed to set pool property "size" on pool ".rgw.root": exit status 2 2022-03-08T18:56:06.997 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:06.996659 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:08.629 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:08.628930 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:10.445 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:10.444029 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:12.327 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:12.326972 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:14.090 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:14.089902 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:15.756 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:15.755975 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:17.751 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:17.751049 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:19.732 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:19.731157 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:21.487 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:21.486508 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:25.104 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:25.103126 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:26.960 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:26.959236 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:28.595 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:28.594464 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:30.466 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:30.465969 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:32.264 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:32.263927 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:32.283 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:32.283181 I | op-osd: waiting... 1 of 1 OSD prepare jobs have finished processing and 9 of 11 OSDs have been updated 2022-03-08T18:56:34.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:34.086847 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:35.606 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:35.606079 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:37.285 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:37.284112 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:39.302 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:39.301836 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:41.141 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:41.140881 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:43.013 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:43.010233 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:45.116 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:45.115998 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:47.044 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:47.043890 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:48.743 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:48.742787 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:50.627 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:50.626635 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:52.733 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:52.733182 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:54.450 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:54.449376 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:56.190 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:56.189746 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:57.998 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:57.997423 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:56:59.721 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:56:59.721208 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:03.333 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:03.332737 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:05.362 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:05.361459 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:07.111 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:07.111320 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:09.077 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:09.076742 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:10.647 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:10.646646 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:12.886 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:12.885555 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:14.728 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:14.728176 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:16.619 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:16.615700 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:18.223 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:18.217694 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:19.874 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:19.873515 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:21.592 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:21.592251 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:23.330 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:23.330010 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:25.235 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:25.235207 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:26.988 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:26.987881 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:28.642 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:28.641715 I | op-osd: OSD 6 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:29.093 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Removed 196959 objects 2022-03-08T18:57:29.093 INFO:tasks.radosbench.radosbench.a.smithi002.stdout:Clean up completed and total clean up time :397.437 2022-03-08T18:57:29.295 INFO:tasks.rook.ceph_manager.ceph:removing pool_name unique_pool_0 2022-03-08T18:57:29.296 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2022-03-08T18:57:30.354 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:30.353977 I | op-osd: updating OSD 3 on PVC "smithi066-lv-3-data-0xl876" 2022-03-08T18:57:30.355 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:30.354009 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-3-data-0xl876" 2022-03-08T18:57:30.718 INFO:teuthology.orchestra.run.smithi002.stderr:pool 'unique_pool_0' removed 2022-03-08T18:57:30.743 DEBUG:teuthology.run_tasks:Unwinding manager install 2022-03-08T18:57:30.756 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits... 2022-03-08T18:57:30.757 DEBUG:teuthology.orchestra.run.smithi002:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits 2022-03-08T18:57:30.760 DEBUG:teuthology.orchestra.run.smithi066:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits 2022-03-08T18:57:30.763 DEBUG:teuthology.orchestra.run.smithi137:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits 2022-03-08T18:57:30.826 DEBUG:teuthology.run_tasks:Unwinding manager rook 2022-03-08T18:57:30.838 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:57:31.560 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:57:31.561 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:57:31.376338Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "mds.foo", "service_type": "mds", "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:57:31.376338Z", "running": 0, "size": 2}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:57:31.376338Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:57:31.376338Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:57:31.376338Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:57:31.376338Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_name": "rbd-mirror", "service_type": "rbd-mirror", "status": {"last_refresh": "2022-03-08T18:57:31.376338Z", "running": 0, "size": 1}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:57:31.376338Z", "running": 0, "size": 1}}] 2022-03-08T18:57:31.580 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch rm mds.foo 2022-03-08T18:57:32.078 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:32.078263 I | op-osd: updating OSD 6 on PVC "smithi066-lv-4-data-04lwmk" 2022-03-08T18:57:32.079 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:32.078295 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-4-data-04lwmk" 2022-03-08T18:57:32.152 INFO:teuthology.orchestra.run.smithi002.stdout:Removed cephfilesystems/foo 2022-03-08T18:57:32.170 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:32.169979 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:57:32.172 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch rm rbd-mirror 2022-03-08T18:57:32.718 INFO:teuthology.orchestra.run.smithi002.stdout:Removed cephrbdmirrors/default-rbd-mirror 2022-03-08T18:57:32.737 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:32.736809 I | op-osd: finished running OSDs in namespace "rook-ceph" 2022-03-08T18:57:32.737 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:32.736835 I | ceph-cluster-controller: done reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:57:32.740 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:32.740314 I | ceph-spec: object "rook-ceph-rbd-mirror-a" matched on delete, reconciling 2022-03-08T18:57:32.741 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch rm rgw.foo 2022-03-08T18:57:32.865 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:32.860556 I | ceph-cluster-controller: reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:57:32.918 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:32.915377 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:57:33.256 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:33.255939 I | ceph-cluster-controller: detecting the ceph image version for image quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae... 2022-03-08T18:57:33.436 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:33.435701 I | op-config: deleting "mds_join_fs" option from the mon configuration database 2022-03-08T18:57:33.926 INFO:teuthology.orchestra.run.smithi002.stdout:Removed cephobjectstores/foo 2022-03-08T18:57:33.949 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:57:33.983 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:33.982087 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:57:35.267 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:57:35.268 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:57:35.087096Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "mds.foo", "service_type": "mds", "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:57:35.087096Z", "running": 0, "size": 2}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:57:35.087096Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:57:35.087096Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:57:35.087096Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:57:35.087096Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:57:35.087096Z", "running": 0, "size": 1}}] 2022-03-08T18:57:35.927 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:35.926632 I | op-config: successfully deleted "mds_join_fs" option from the mon configuration database 2022-03-08T18:57:35.927 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:35.926654 I | op-mds: successfully deleted mds config for "mds.foo-a" in mon configuration database 2022-03-08T18:57:35.928 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:35.926658 I | cephclient: deleting ceph auth "mds.foo-a" 2022-03-08T18:57:37.769 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:37.769061 I | op-mds: successfully deleted mds CephX key for "mds.foo-a" 2022-03-08T18:57:38.110 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:38.109551 I | op-config: deleting "mds_join_fs" option from the mon configuration database 2022-03-08T18:57:41.088 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:41.087295 I | op-config: successfully deleted "mds_join_fs" option from the mon configuration database 2022-03-08T18:57:41.088 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:41.087325 I | op-mds: successfully deleted mds config for "mds.foo-b" in mon configuration database 2022-03-08T18:57:41.089 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:41.087335 I | cephclient: deleting ceph auth "mds.foo-b" 2022-03-08T18:57:41.106 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:41.105120 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T18:57:41.098+0000 7fbf54848340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T18:57:41.106 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T18:57:41.098+0000 7fbf54848340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T18:57:41.106 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T18:57:41.098+0000 7fbf54848340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T18:57:41.107 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T18:57:41.107 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:41.105152 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T18:57:41.098+0000 7fbf54848340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T18:57:41.107 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T18:57:41.098+0000 7fbf54848340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T18:57:41.108 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T18:57:41.098+0000 7fbf54848340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T18:57:41.108 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T18:57:42.040 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:42.039811 I | ceph-cluster-controller: detected ceph image version: "17.0.0-10961 quincy" 2022-03-08T18:57:42.041 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:42.039827 I | ceph-cluster-controller: validating ceph version from provided image 2022-03-08T18:57:42.041 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:42.039834 W | ceph-cluster-controller: unsupported ceph version detected: "17.0.0-10961 quincy", pursuing 2022-03-08T18:57:42.108 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:42.108078 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:57:42.262 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:42.261552 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:57:42.308 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:42.307226 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:57:42.718 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:42.715616 I | ceph-cluster-controller: cluster "rook-ceph": version "17.0.0-10961 quincy" detected for image "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae" 2022-03-08T18:57:42.817 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:42.817104 I | op-mds: successfully deleted mds CephX key for "mds.foo-b" 2022-03-08T18:57:42.818 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:42.817142 I | ceph-file-controller: downing filesystem "foo" 2022-03-08T18:57:42.892 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:42.891728 I | op-mon: start running mons 2022-03-08T18:57:42.945 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:42.944977 I | op-mon: parsing mon endpoints: a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 2022-03-08T18:57:43.142 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:43.141999 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:a=10.101.136.169:6789,b=10.110.64.66:6789,c=10.109.10.123:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:57:43.150 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:43.150348 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:57:43.152 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:43.150524 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:57:43.985 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:43.984260 I | op-mon: targeting the mon count 3 2022-03-08T18:57:43.991 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:43.991828 I | op-config: setting "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:57:44.302 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:44.301622 I | ceph-file-controller: downed filesystem "foo" 2022-03-08T18:57:44.944 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:44.943965 I | op-config: successfully set "global"="mon allow pool delete"="true" option to the mon configuration database 2022-03-08T18:57:44.945 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:44.943987 I | op-config: setting "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:57:45.292 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:45.292001 I | op-config: successfully set "global"="mon cluster log file"="" option to the mon configuration database 2022-03-08T18:57:45.293 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:45.292024 I | op-config: setting "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:57:45.294 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:57:45.737 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:45.736507 I | op-config: successfully set "global"="mon allow pool size one"="true" option to the mon configuration database 2022-03-08T18:57:45.737 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:45.736543 I | op-config: setting "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:57:46.636 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:46.635751 I | op-config: successfully set "global"="log to file"="false" option to the mon configuration database 2022-03-08T18:57:46.636 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:46.635771 I | op-config: setting "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:57:46.638 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:57:46.638 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:57:45.929152Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "mds.foo", "service_type": "mds", "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:57:45.929152Z", "running": 0, "size": 2}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:57:45.929152Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:57:45.929152Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:57:45.929152Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:57:45.929152Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:57:45.929152Z", "running": 0, "size": 1}}] 2022-03-08T18:57:47.015 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:47.014394 I | op-config: successfully set "global"="rbd_default_features"="3" option to the mon configuration database 2022-03-08T18:57:47.015 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:47.014427 I | op-config: deleting "log file" option from the mon configuration database 2022-03-08T18:57:47.377 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:47.377232 I | op-config: successfully deleted "log file" option from the mon configuration database 2022-03-08T18:57:47.378 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:47.377278 I | op-mon: checking for basic quorum with existing mons 2022-03-08T18:57:47.554 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:47.553630 I | op-mon: mon "a" endpoint is [v2:10.101.136.169:3300,v1:10.101.136.169:6789] 2022-03-08T18:57:47.671 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:47.671004 I | op-mon: mon "b" endpoint is [v2:10.110.64.66:3300,v1:10.110.64.66:6789] 2022-03-08T18:57:47.783 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:47.783409 I | op-mon: mon "c" endpoint is [v2:10.109.10.123:3300,v1:10.109.10.123:6789] 2022-03-08T18:57:48.460 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:48.459940 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.101.136.169:6789","10.110.64.66:6789","10.109.10.123:6789"]}] data:b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 mapping:{"node":{"a":{"Name":"smithi137","Hostname":"smithi137","Address":"172.21.15.137"},"b":{"Name":"smithi066","Hostname":"smithi066","Address":"172.21.15.66"},"c":{"Name":"smithi002","Hostname":"smithi002","Address":"172.21.15.2"}}} maxMonId:2] 2022-03-08T18:57:48.999 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:48.998634 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2022-03-08T18:57:48.999 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:48.998840 I | cephclient: generated admin config in /var/lib/rook/rook-ceph 2022-03-08T18:57:49.476 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:49.475652 I | op-mon: deployment for mon rook-ceph-mon-a already exists. updating if needed 2022-03-08T18:57:49.485 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:49.485591 I | op-k8sutil: deployment "rook-ceph-mon-a" did not change, nothing to update 2022-03-08T18:57:49.486 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:49.485623 I | op-mon: waiting for mon quorum with [a b c] 2022-03-08T18:57:49.999 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:49.998467 I | op-mon: mons running: [a b c] 2022-03-08T18:57:50.446 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:50.446250 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:57:50.454 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:50.454305 I | op-mon: deployment for mon rook-ceph-mon-b already exists. updating if needed 2022-03-08T18:57:50.462 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:50.462396 I | op-k8sutil: deployment "rook-ceph-mon-b" did not change, nothing to update 2022-03-08T18:57:50.463 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:50.462420 I | op-mon: waiting for mon quorum with [a b c] 2022-03-08T18:57:50.608 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:50.608214 I | op-mon: mons running: [a b c] 2022-03-08T18:57:51.027 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:51.027373 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:57:51.082 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:51.078183 I | op-mon: deployment for mon rook-ceph-mon-c already exists. updating if needed 2022-03-08T18:57:51.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:51.086754 I | op-k8sutil: deployment "rook-ceph-mon-c" did not change, nothing to update 2022-03-08T18:57:51.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:51.086777 I | op-mon: waiting for mon quorum with [a b c] 2022-03-08T18:57:51.208 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:51.208424 I | op-mon: mons running: [a b c] 2022-03-08T18:57:51.297 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:51.297131 I | cephclient: no images/snapshosts present in pool "foo-metadata" 2022-03-08T18:57:51.297 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:51.297151 I | cephclient: purging pool "foo-metadata" (id=4) 2022-03-08T18:57:51.660 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:51.660305 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:57:51.661 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:51.660323 I | op-mon: mons created: 3 2022-03-08T18:57:52.073 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:52.072635 I | op-mon: waiting for mon quorum with [a b c] 2022-03-08T18:57:52.806 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:52.806121 I | op-mon: mons running: [a b c] 2022-03-08T18:57:53.241 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:53.240296 I | op-mon: Monitors in quorum: [a b c] 2022-03-08T18:57:53.241 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:53.240341 I | cephclient: getting or creating ceph auth key "client.csi-rbd-provisioner" 2022-03-08T18:57:53.712 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:53.712213 I | cephclient: getting or creating ceph auth key "client.csi-rbd-node" 2022-03-08T18:57:54.130 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:54.129714 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-provisioner" 2022-03-08T18:57:54.371 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:54.371008 I | cephclient: purge completed for pool "foo-metadata" 2022-03-08T18:57:54.604 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:54.603991 I | cephclient: getting or creating ceph auth key "client.csi-cephfs-node" 2022-03-08T18:57:54.783 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:54.782625 I | cephclient: no images/snapshosts present in pool "foo-data0" 2022-03-08T18:57:54.784 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:54.782649 I | cephclient: purging pool "foo-data0" (id=10) 2022-03-08T18:57:55.050 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:55.050340 I | ceph-csi: created kubernetes csi secrets for cluster "rook-ceph" 2022-03-08T18:57:55.051 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:55.050371 I | cephclient: getting or creating ceph auth key "client.crash" 2022-03-08T18:57:55.466 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:55.466041 I | ceph-crashcollector-controller: created kubernetes crash collector secret for cluster "rook-ceph" 2022-03-08T18:57:55.894 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:55.893738 I | cephclient: successfully enabled msgr2 protocol 2022-03-08T18:57:55.894 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:55.893783 I | cephclient: create rbd-mirror bootstrap peer token "client.rbd-mirror-peer" 2022-03-08T18:57:55.895 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:55.893797 I | cephclient: getting or creating ceph auth key "client.rbd-mirror-peer" 2022-03-08T18:57:56.329 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:56.328437 I | cephclient: successfully created rbd-mirror bootstrap peer token for cluster "rook-ceph" 2022-03-08T18:57:56.361 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:56.361465 I | op-mgr: start running mgr 2022-03-08T18:57:56.362 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:56.361501 I | cephclient: getting or creating ceph auth key "mgr.a" 2022-03-08T18:57:56.664 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:57:56.866 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:56.865573 I | op-mgr: deployment for mgr rook-ceph-mgr-a already exists. updating if needed 2022-03-08T18:57:56.871 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:56.871222 I | op-k8sutil: deployment "rook-ceph-mgr-a" did not change, nothing to update 2022-03-08T18:57:56.873 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:56.873159 I | op-mgr: setting services to point to mgr "a" 2022-03-08T18:57:57.073 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.072927 I | op-osd: start running osds in namespace "rook-ceph" 2022-03-08T18:57:57.073 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.072957 I | op-osd: wait timeout for healthy OSDs during upgrade or restart is "10m0s" 2022-03-08T18:57:57.153 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.153577 I | op-osd: start provisioning the OSDs on PVCs, if needed 2022-03-08T18:57:57.162 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159786 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-1" 2022-03-08T18:57:57.162 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159828 I | op-osd: OSD PVC "smithi002-lv-1-data-0cg8pd" already exists 2022-03-08T18:57:57.163 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159841 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-2" 2022-03-08T18:57:57.163 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159852 I | op-osd: OSD PVC "smithi002-lv-2-data-09sjq5" already exists 2022-03-08T18:57:57.163 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159862 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-3" 2022-03-08T18:57:57.163 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159872 I | op-osd: OSD PVC "smithi002-lv-3-data-05dmhp" already exists 2022-03-08T18:57:57.163 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159884 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi002-lv-4" 2022-03-08T18:57:57.164 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159894 I | op-osd: OSD PVC "smithi002-lv-4-data-0pb5hz" already exists 2022-03-08T18:57:57.164 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159901 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-2" 2022-03-08T18:57:57.164 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159911 I | op-osd: OSD PVC "smithi066-lv-2-data-0jsgjv" already exists 2022-03-08T18:57:57.164 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159918 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-3" 2022-03-08T18:57:57.165 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159929 I | op-osd: OSD PVC "smithi066-lv-3-data-0xl876" already exists 2022-03-08T18:57:57.165 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159938 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-4" 2022-03-08T18:57:57.165 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159949 I | op-osd: OSD PVC "smithi066-lv-4-data-04lwmk" already exists 2022-03-08T18:57:57.165 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159958 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-1" 2022-03-08T18:57:57.165 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159967 I | op-osd: OSD PVC "smithi137-lv-1-data-0cmhmp" already exists 2022-03-08T18:57:57.166 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159976 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-2" 2022-03-08T18:57:57.166 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159985 I | op-osd: OSD PVC "smithi137-lv-2-data-0whzhj" already exists 2022-03-08T18:57:57.166 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.159996 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-3" 2022-03-08T18:57:57.166 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.160009 I | op-osd: OSD PVC "smithi137-lv-3-data-0kg6xk" already exists 2022-03-08T18:57:57.167 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.160018 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi137-lv-4" 2022-03-08T18:57:57.167 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.160027 I | op-osd: OSD PVC "smithi137-lv-4-data-0zchnd" already exists 2022-03-08T18:57:57.167 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.160036 I | op-osd: verifying PVCs exist for 1 OSDs in device set "smithi066-lv-1" 2022-03-08T18:57:57.167 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.160045 I | op-osd: OSD PVC "smithi066-lv-1-data-0snd8j" already exists 2022-03-08T18:57:57.191 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190653 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:57:57.192 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190674 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-2-data-09sjq5" 2022-03-08T18:57:57.192 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190678 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-3-data-05dmhp" 2022-03-08T18:57:57.192 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190681 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:57:57.192 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190684 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:57:57.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190688 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-3-data-0xl876" 2022-03-08T18:57:57.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190692 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-4-data-04lwmk" 2022-03-08T18:57:57.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190697 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:57:57.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190702 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-2-data-0whzhj" 2022-03-08T18:57:57.194 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190709 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:57:57.194 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190714 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-4-data-0zchnd" 2022-03-08T18:57:57.194 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190723 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-1-data-0snd8j" 2022-03-08T18:57:57.194 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190731 I | op-osd: start provisioning the OSDs on nodes, if needed 2022-03-08T18:57:57.194 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.190737 I | op-osd: no nodes are defined for configuring OSDs on raw devices 2022-03-08T18:57:57.336 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:57:57.336 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:57:57.176090Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "mds.foo", "service_type": "mds", "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:57:57.176090Z", "running": 0, "size": 2}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:57:57.176090Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:57:57.176090Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:57:57.176090Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:57:57.176090Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:57:57.176090Z", "running": 0, "size": 1}}] 2022-03-08T18:57:57.340 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.340320 I | cephclient: purge completed for pool "foo-data0" 2022-03-08T18:57:57.340 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.340349 I | ceph-spec: removing finalizer "cephfilesystem.ceph.rook.io" on "foo" 2022-03-08T18:57:57.364 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.364423 I | ceph-spec: object "rook-ceph-mds-foo-a-keyring" matched on delete, reconciling 2022-03-08T18:57:57.368 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.368925 I | ceph-spec: object "rook-ceph-mds-foo-a" matched on delete, reconciling 2022-03-08T18:57:57.377 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.376419 I | ceph-spec: object "rook-ceph-mds-foo-b-keyring" matched on delete, reconciling 2022-03-08T18:57:57.386 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:57.385809 I | ceph-spec: object "rook-ceph-mds-foo-b" matched on delete, reconciling 2022-03-08T18:57:59.055 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:59.049258 I | op-osd: OSD 0 is not ok-to-stop. will try updating it again later 2022-03-08T18:57:59.326 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:59.326006 I | op-mgr: successful modules: mgr module(s) from the spec 2022-03-08T18:57:59.332 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:59.331861 I | op-mgr: successful modules: dashboard 2022-03-08T18:57:59.336 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:59.336102 I | op-mgr: successful modules: prometheus 2022-03-08T18:57:59.989 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:57:59.989085 I | op-mgr: successful modules: balancer 2022-03-08T18:58:00.797 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:00.797480 I | op-osd: updating OSD 1 on PVC "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:58:00.798 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:00.797510 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-1-data-0cmhmp" 2022-03-08T18:58:01.797 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:01.797009 I | op-mgr: successful modules: orchestrator modules 2022-03-08T18:58:03.075 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:03.071123 I | op-osd: updating OSD 10 on PVC "smithi137-lv-4-data-0zchnd" 2022-03-08T18:58:03.075 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:03.071153 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-4-data-0zchnd" 2022-03-08T18:58:04.957 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:04.956627 I | op-osd: updating OSD 11 on PVC "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:58:04.958 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:04.956657 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-4-data-0pb5hz" 2022-03-08T18:58:06.613 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:06.612353 I | op-osd: updating OSD 2 on PVC "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:58:06.613 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:06.612385 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-1-data-0cg8pd" 2022-03-08T18:58:07.357 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:58:08.073 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:58:08.073 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:58:07.862203Z", "running": 5, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:58:07.862203Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:58:07.862203Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:58:07.862203Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:58:07.862203Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:58:07.862203Z", "running": 0, "size": 1}}] 2022-03-08T18:58:08.316 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:08.316285 I | op-osd: OSD 3 is not ok-to-stop. will try updating it again later 2022-03-08T18:58:09.873 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:09.872974 I | op-osd: updating OSD 4 on PVC "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:58:09.873 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:09.873010 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-2-data-0jsgjv" 2022-03-08T18:58:11.592 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:11.592198 I | op-osd: updating OSD 5 on PVC "smithi002-lv-2-data-09sjq5" 2022-03-08T18:58:11.593 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:11.592217 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-2-data-09sjq5" 2022-03-08T18:58:13.440 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:13.439485 I | op-osd: updating OSD 6 on PVC "smithi066-lv-4-data-04lwmk" 2022-03-08T18:58:13.440 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:13.439533 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-4-data-04lwmk" 2022-03-08T18:58:15.068 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:15.067651 I | op-osd: updating OSD 7 on PVC "smithi137-lv-2-data-0whzhj" 2022-03-08T18:58:15.068 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:15.067687 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-2-data-0whzhj" 2022-03-08T18:58:16.790 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:16.790411 I | op-osd: updating OSD 8 on PVC "smithi002-lv-3-data-05dmhp" 2022-03-08T18:58:16.791 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:16.790440 I | op-osd: OSD will have its main bluestore block on "smithi002-lv-3-data-05dmhp" 2022-03-08T18:58:18.096 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:58:18.377 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:18.377463 I | op-osd: updating OSD 9 on PVC "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:58:18.378 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:18.377489 I | op-osd: OSD will have its main bluestore block on "smithi137-lv-3-data-0kg6xk" 2022-03-08T18:58:18.870 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:58:18.870 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:58:18.681834Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:58:18.681834Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:58:18.681834Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:58:18.681834Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:58:18.681834Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:58:18.681834Z", "running": 0, "size": 1}}] 2022-03-08T18:58:20.048 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:20.048516 I | op-osd: updating OSD 0 on PVC "smithi066-lv-1-data-0snd8j" 2022-03-08T18:58:20.049 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:20.048538 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-1-data-0snd8j" 2022-03-08T18:58:21.649 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:21.649093 I | op-osd: updating OSD 3 on PVC "smithi066-lv-3-data-0xl876" 2022-03-08T18:58:21.650 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:21.649126 I | op-osd: OSD will have its main bluestore block on "smithi066-lv-3-data-0xl876" 2022-03-08T18:58:22.310 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:22.310408 I | op-osd: finished running OSDs in namespace "rook-ceph" 2022-03-08T18:58:22.311 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 18:58:22.310431 I | ceph-cluster-controller: done reconciling ceph cluster in namespace "rook-ceph" 2022-03-08T18:58:28.892 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:58:29.982 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:58:29.982 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:58:29.787955Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:58:29.787955Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:58:29.787955Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:58:29.787955Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:58:29.787955Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:58:29.787955Z", "running": 0, "size": 1}}] 2022-03-08T18:58:40.006 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:58:43.404 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:58:43.404 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:58:43.225113Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:58:43.225113Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:58:43.225113Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:58:43.225113Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:58:43.225113Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:58:43.225113Z", "running": 0, "size": 1}}] 2022-03-08T18:58:53.432 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:58:54.166 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:58:54.167 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:58:53.994753Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:58:53.994753Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:58:53.994753Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:58:53.994753Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:58:53.994753Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:58:53.994753Z", "running": 0, "size": 1}}] 2022-03-08T18:59:04.195 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:59:04.939 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:59:04.940 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:59:04.764598Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:59:04.764598Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:59:04.764598Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:59:04.764598Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:59:04.764598Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:59:04.764598Z", "running": 0, "size": 1}}] 2022-03-08T18:59:14.965 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:59:15.646 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:59:15.646 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:59:15.477764Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:59:15.477764Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:59:15.477764Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:59:15.477764Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:59:15.477764Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:59:15.477764Z", "running": 0, "size": 1}}] 2022-03-08T18:59:25.672 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:59:26.377 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:59:26.377 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:59:26.218152Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:59:26.218152Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:59:26.218152Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:59:26.218152Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:59:26.218152Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:59:26.218152Z", "running": 0, "size": 1}}] 2022-03-08T18:59:36.398 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:59:37.084 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:59:37.085 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:59:36.919815Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:59:36.919815Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:59:36.919815Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:59:36.919815Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:59:36.919815Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:59:36.919815Z", "running": 0, "size": 1}}] 2022-03-08T18:59:47.110 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:59:47.859 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:59:47.859 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:59:47.638220Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:59:47.638220Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:59:47.638220Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:59:47.638220Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:59:47.638220Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:59:47.638220Z", "running": 0, "size": 1}}] 2022-03-08T18:59:57.892 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T18:59:58.591 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T18:59:58.591 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T18:59:58.422386Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T18:59:58.422386Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T18:59:58.422386Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T18:59:58.422386Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T18:59:58.422386Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T18:59:58.422386Z", "running": 0, "size": 1}}] 2022-03-08T19:00:08.614 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:00:09.364 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:00:09.364 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:00:09.137973Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:00:09.137973Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:00:09.137973Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:00:09.137973Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:00:09.137973Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:00:09.137973Z", "running": 0, "size": 1}}] 2022-03-08T19:00:19.391 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:00:20.363 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:00:20.363 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:00:19.922174Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:00:19.922174Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:00:19.922174Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:00:19.922174Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:00:19.922174Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:00:19.922174Z", "running": 0, "size": 1}}] 2022-03-08T19:00:30.396 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:00:31.200 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:00:31.201 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:00:31.025914Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:00:31.025914Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:00:31.025914Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:00:31.025914Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:00:31.025914Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:00:31.025914Z", "running": 0, "size": 1}}] 2022-03-08T19:00:41.227 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:00:41.966 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:00:41.967 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:00:41.762403Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:00:41.762403Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:00:41.762403Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:00:41.762403Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:00:41.762403Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:00:41.762403Z", "running": 0, "size": 1}}] 2022-03-08T19:00:51.995 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:00:52.856 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:00:52.856 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:00:52.561797Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:00:52.561797Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:00:52.561797Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:00:52.561797Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:00:52.561797Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:00:52.561797Z", "running": 0, "size": 1}}] 2022-03-08T19:01:02.884 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:01:03.624 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:01:03.625 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:01:03.420563Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:01:03.420563Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:01:03.420563Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:01:03.420563Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:01:03.420563Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:01:03.420563Z", "running": 0, "size": 1}}] 2022-03-08T19:01:13.658 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:01:14.380 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:01:14.380 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:01:14.196120Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:01:14.196120Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:01:14.196120Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:01:14.196120Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:01:14.196120Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:01:14.196120Z", "running": 0, "size": 1}}] 2022-03-08T19:01:24.408 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:01:25.103 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:01:25.104 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:01:24.936987Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:01:24.936987Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:01:24.936987Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:01:24.936987Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:01:24.936987Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:01:24.936987Z", "running": 0, "size": 1}}] 2022-03-08T19:01:34.276 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:01:34.276112 I | op-mon: parsing mon endpoints: b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 2022-03-08T19:01:34.922 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:01:34.922608 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T19:01:34.911+0000 7fdfa8575340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T19:01:34.923 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:01:34.911+0000 7fdfa8575340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T19:01:34.923 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:01:34.911+0000 7fdfa8575340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T19:01:34.923 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T19:01:34.924 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:01:34.922638 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T19:01:34.911+0000 7fdfa8575340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T19:01:34.924 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:01:34.911+0000 7fdfa8575340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T19:01:34.924 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:01:34.911+0000 7fdfa8575340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T19:01:34.925 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T19:01:35.130 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:01:35.865 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:01:35.865 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:01:35.688033Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:01:35.688033Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:01:35.688033Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:01:35.688033Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:01:35.688033Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:01:35.688033Z", "running": 0, "size": 1}}] 2022-03-08T19:01:45.894 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:01:46.625 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:01:46.625 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:01:46.453567Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:01:46.453567Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:01:46.453567Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:01:46.453567Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:01:46.453567Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:01:46.453567Z", "running": 0, "size": 1}}] 2022-03-08T19:01:56.649 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:01:57.331 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:01:57.332 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:01:57.150110Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:01:57.150110Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:01:57.150110Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:01:57.150110Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:01:57.150110Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:01:57.150110Z", "running": 0, "size": 1}}] 2022-03-08T19:02:07.359 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:02:08.069 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:02:08.069 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:02:07.892720Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:02:07.892720Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:02:07.892720Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:02:07.892720Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:02:07.892720Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:02:07.892720Z", "running": 0, "size": 1}}] 2022-03-08T19:02:18.095 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:02:18.864 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:02:18.865 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:02:18.635305Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:02:18.635305Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:02:18.635305Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:02:18.635305Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:02:18.635305Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:02:18.635305Z", "running": 0, "size": 1}}] 2022-03-08T19:02:28.891 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:02:29.541 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:02:29.541 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:02:29.367102Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:02:29.367102Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:02:29.367102Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:02:29.367102Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:02:29.367102Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:02:29.367102Z", "running": 0, "size": 1}}] 2022-03-08T19:02:39.568 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:02:40.286 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:02:40.287 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:02:40.110418Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:02:40.110418Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:02:40.110418Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:02:40.110418Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:02:40.110418Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:02:40.110418Z", "running": 0, "size": 1}}] 2022-03-08T19:02:50.312 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:02:51.038 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:02:51.038 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:02:50.830085Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:02:50.830085Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:02:50.830085Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:02:50.830085Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:02:50.830085Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:02:50.830085Z", "running": 0, "size": 1}}] 2022-03-08T19:03:01.067 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:03:01.821 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:03:01.822 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:03:01.589055Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:03:01.589055Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:03:01.589055Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:03:01.589055Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:03:01.589055Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:03:01.589055Z", "running": 0, "size": 1}}] 2022-03-08T19:03:11.850 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:03:12.548 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:03:12.549 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:03:12.376656Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:03:12.376656Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:03:12.376656Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:03:12.376656Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:03:12.376656Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:03:12.376656Z", "running": 0, "size": 1}}] 2022-03-08T19:03:22.570 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:03:23.287 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:03:23.287 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:03:23.118069Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:03:23.118069Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:03:23.118069Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:03:23.118069Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:03:23.118069Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:03:23.118069Z", "running": 0, "size": 1}}] 2022-03-08T19:03:33.312 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:03:34.031 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:03:34.031 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:03:33.841739Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:03:33.841739Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:03:33.841739Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:03:33.841739Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:03:33.841739Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:03:33.841739Z", "running": 0, "size": 1}}] 2022-03-08T19:03:44.060 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:03:44.735 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:03:44.735 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:03:44.563132Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:03:44.563132Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:03:44.563132Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:03:44.563132Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:03:44.563132Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:03:44.563132Z", "running": 0, "size": 1}}] 2022-03-08T19:03:54.763 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:03:55.486 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:03:55.486 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:03:55.303229Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:03:55.303229Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:03:55.303229Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:03:55.303229Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:03:55.303229Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:03:55.303229Z", "running": 0, "size": 1}}] 2022-03-08T19:04:05.513 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:04:06.370 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:04:06.370 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:04:06.136748Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:04:06.136748Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:04:06.136748Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:04:06.136748Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:04:06.136748Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:04:06.136748Z", "running": 0, "size": 1}}] 2022-03-08T19:04:16.396 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:04:17.102 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:04:17.102 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:04:16.944691Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:04:16.944691Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:04:16.944691Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:04:16.944691Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:04:16.944691Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:04:16.944691Z", "running": 0, "size": 1}}] 2022-03-08T19:04:27.128 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:04:27.901 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:04:27.901 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:04:27.728712Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:04:27.728712Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:04:27.728712Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:04:27.728712Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:04:27.728712Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:04:27.728712Z", "running": 0, "size": 1}}] 2022-03-08T19:04:37.930 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:04:38.641 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:04:38.642 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:04:38.481686Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:04:38.481686Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:04:38.481686Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:04:38.481686Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:04:38.481686Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:04:38.481686Z", "running": 0, "size": 1}}] 2022-03-08T19:04:48.662 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:04:49.417 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:04:49.418 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:04:49.249623Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:04:49.249623Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:04:49.249623Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:04:49.249623Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:04:49.249623Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:04:49.249623Z", "running": 0, "size": 1}}] 2022-03-08T19:04:59.443 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:05:00.161 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:05:00.161 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:04:59.983498Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:04:59.983498Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:04:59.983498Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:04:59.983498Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:04:59.983498Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:04:59.983498Z", "running": 0, "size": 1}}] 2022-03-08T19:05:10.186 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:05:10.917 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:05:10.918 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:05:10.733909Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:05:10.733909Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:05:10.733909Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:05:10.733909Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:05:10.733909Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:05:10.733909Z", "running": 0, "size": 1}}] 2022-03-08T19:05:20.949 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:05:21.657 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:05:21.657 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:05:21.479315Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:05:21.479315Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:05:21.479315Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:05:21.479315Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:05:21.479315Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:05:21.479315Z", "running": 0, "size": 1}}] 2022-03-08T19:05:31.684 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:05:32.425 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:05:32.426 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:05:32.223199Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:05:32.223199Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:05:32.223199Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:05:32.223199Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:05:32.223199Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:05:32.223199Z", "running": 0, "size": 1}}] 2022-03-08T19:05:42.456 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:05:43.231 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:05:43.231 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:05:42.991497Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:05:42.991497Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:05:42.991497Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:05:42.991497Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:05:42.991497Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:05:42.991497Z", "running": 0, "size": 1}}] 2022-03-08T19:05:53.260 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:05:54.009 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:05:54.009 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:05:53.796829Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:05:53.796829Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:05:53.796829Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:05:53.796829Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:05:53.796829Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:05:53.796829Z", "running": 0, "size": 1}}] 2022-03-08T19:06:04.040 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:06:04.761 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:06:04.762 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:06:04.576196Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:06:04.576196Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:06:04.576196Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:06:04.576196Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:06:04.576196Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:06:04.576196Z", "running": 0, "size": 1}}] 2022-03-08T19:06:14.788 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:06:15.517 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:06:15.518 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:06:15.316444Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:06:15.316444Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:06:15.316444Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:06:15.316444Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:06:15.316444Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:06:15.316444Z", "running": 0, "size": 1}}] 2022-03-08T19:06:25.544 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:06:26.262 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:06:26.263 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:06:26.080829Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:06:26.080829Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:06:26.080829Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:06:26.080829Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:06:26.080829Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:06:26.080829Z", "running": 0, "size": 1}}] 2022-03-08T19:06:36.290 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:06:37.027 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:06:37.027 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:06:36.817317Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:06:36.817317Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:06:36.817317Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:06:36.817317Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:06:36.817317Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:06:36.817317Z", "running": 0, "size": 1}}] 2022-03-08T19:06:47.054 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:06:47.810 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:06:47.810 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:06:47.637279Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:06:47.637279Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:06:47.637279Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:06:47.637279Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:06:47.637279Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:06:47.637279Z", "running": 0, "size": 1}}] 2022-03-08T19:06:57.833 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:06:58.580 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:06:58.581 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:06:58.365555Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:06:58.365555Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:06:58.365555Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:06:58.365555Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:06:58.365555Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:06:58.365555Z", "running": 0, "size": 1}}] 2022-03-08T19:07:08.605 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:07:09.362 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:07:09.363 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:07:09.184976Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:07:09.184976Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:07:09.184976Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:07:09.184976Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:07:09.184976Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:07:09.184976Z", "running": 0, "size": 1}}] 2022-03-08T19:07:19.390 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:07:20.155 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:07:20.156 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:07:19.970343Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:07:19.970343Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:07:19.970343Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:07:19.970343Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:07:19.970343Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:07:19.970343Z", "running": 0, "size": 1}}] 2022-03-08T19:07:30.185 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:07:30.896 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:07:30.897 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:07:30.719407Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:07:30.719407Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:07:30.719407Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:07:30.719407Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:07:30.719407Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:07:30.719407Z", "running": 0, "size": 1}}] 2022-03-08T19:07:40.922 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:07:41.654 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:07:41.655 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:07:41.479805Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:07:41.479805Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:07:41.479805Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:07:41.479805Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:07:41.479805Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:07:41.479805Z", "running": 0, "size": 1}}] 2022-03-08T19:07:51.684 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:07:52.435 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:07:52.436 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:07:52.257547Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:07:52.257547Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:07:52.257547Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:07:52.257547Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:07:52.257547Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:07:52.257547Z", "running": 0, "size": 1}}] 2022-03-08T19:08:02.463 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:08:03.171 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:08:03.172 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:08:03.011023Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:08:03.011023Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:08:03.011023Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:08:03.011023Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:08:03.011023Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:08:03.011023Z", "running": 0, "size": 1}}] 2022-03-08T19:08:13.199 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:08:14.018 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:08:14.019 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:08:13.791641Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:08:13.791641Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:08:13.791641Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:08:13.791641Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:08:13.791641Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:08:13.791641Z", "running": 0, "size": 1}}] 2022-03-08T19:08:24.044 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:08:24.786 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:08:24.787 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:08:24.618797Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:08:24.618797Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:08:24.618797Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:08:24.618797Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:08:24.618797Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:08:24.618797Z", "running": 0, "size": 1}}] 2022-03-08T19:08:34.814 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:08:36.096 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:08:36.096 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:08:35.818272Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:08:35.818272Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:08:35.818272Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:08:35.818272Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:08:35.818272Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:08:35.818272Z", "running": 0, "size": 1}}] 2022-03-08T19:08:46.123 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:08:49.369 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:08:49.369 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:08:49.158680Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:08:49.158680Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:08:49.158680Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:08:49.158680Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:08:49.158680Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:08:49.158680Z", "running": 0, "size": 1}}] 2022-03-08T19:08:59.405 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:09:00.093 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:09:00.094 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:08:59.925661Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:08:59.925661Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:08:59.925661Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:08:59.925661Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:08:59.925661Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:08:59.925661Z", "running": 0, "size": 1}}] 2022-03-08T19:09:10.124 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:09:10.881 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:09:10.882 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:09:10.708713Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:09:10.708713Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:09:10.708713Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:09:10.708713Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:09:10.708713Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:09:10.708713Z", "running": 0, "size": 1}}] 2022-03-08T19:09:20.908 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:09:21.661 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:09:21.662 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:09:21.494654Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:09:21.494654Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:09:21.494654Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:09:21.494654Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:09:21.494654Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:09:21.494654Z", "running": 0, "size": 1}}] 2022-03-08T19:09:31.690 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:09:32.381 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:09:32.382 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:09:32.222467Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:09:32.222467Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:09:32.222467Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:09:32.222467Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:09:32.222467Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:09:32.222467Z", "running": 0, "size": 1}}] 2022-03-08T19:09:42.409 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:09:43.127 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:09:43.127 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:09:42.952598Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:09:42.952598Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:09:42.952598Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:09:42.952598Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:09:42.952598Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:09:42.952598Z", "running": 0, "size": 1}}] 2022-03-08T19:09:53.154 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:09:53.939 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:09:53.939 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:09:53.724957Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:09:53.724957Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:09:53.724957Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:09:53.724957Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:09:53.724957Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:09:53.724957Z", "running": 0, "size": 1}}] 2022-03-08T19:10:03.966 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:10:04.701 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:10:04.701 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:10:04.502371Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:10:04.502371Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:10:04.502371Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:10:04.502371Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:10:04.502371Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:10:04.502371Z", "running": 0, "size": 1}}] 2022-03-08T19:10:14.725 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:10:15.440 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:10:15.441 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:10:15.265699Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:10:15.265699Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:10:15.265699Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:10:15.265699Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:10:15.265699Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:10:15.265699Z", "running": 0, "size": 1}}] 2022-03-08T19:10:25.465 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:10:26.168 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:10:26.168 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:10:25.993029Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:10:25.993029Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:10:25.993029Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:10:25.993029Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:10:25.993029Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:10:25.993029Z", "running": 0, "size": 1}}] 2022-03-08T19:10:36.197 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:10:36.924 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:10:36.925 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:10:36.745765Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:10:36.745765Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:10:36.745765Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:10:36.745765Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:10:36.745765Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:10:36.745765Z", "running": 0, "size": 1}}] 2022-03-08T19:10:46.950 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:10:47.669 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:10:47.670 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:10:47.489138Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:10:47.489138Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:10:47.489138Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:10:47.489138Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:10:47.489138Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:10:47.489138Z", "running": 0, "size": 1}}] 2022-03-08T19:10:57.693 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:10:58.755 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:10:58.755 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:10:58.249924Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:10:58.249924Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:10:58.249924Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:10:58.249924Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:10:58.249924Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:10:58.249924Z", "running": 0, "size": 1}}] 2022-03-08T19:11:08.783 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:11:09.521 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:11:09.522 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:11:09.299337Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:11:09.299337Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:11:09.299337Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:11:09.299337Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:11:09.299337Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:11:09.299337Z", "running": 0, "size": 1}}] 2022-03-08T19:11:19.548 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:11:20.435 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:11:20.435 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:11:20.159530Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:11:20.159530Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:11:20.159530Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:11:20.159530Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:11:20.159530Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:11:20.159530Z", "running": 0, "size": 1}}] 2022-03-08T19:11:30.461 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:11:31.167 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:11:31.167 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:11:30.999745Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:11:30.999745Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:11:30.999745Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:11:30.999745Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:11:30.999745Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:11:30.999745Z", "running": 0, "size": 1}}] 2022-03-08T19:11:41.198 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:11:41.956 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:11:41.957 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:11:41.755395Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:11:41.755395Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:11:41.755395Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:11:41.755395Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:11:41.755395Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:11:41.755395Z", "running": 0, "size": 1}}] 2022-03-08T19:11:51.985 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:11:52.670 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:11:52.670 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:11:52.490846Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:11:52.490846Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:11:52.490846Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:11:52.490846Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:11:52.490846Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:11:52.490846Z", "running": 0, "size": 1}}] 2022-03-08T19:12:02.698 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:12:03.428 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:12:03.428 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:12:03.252607Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:12:03.252607Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:12:03.252607Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:12:03.252607Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:12:03.252607Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:12:03.252607Z", "running": 0, "size": 1}}] 2022-03-08T19:12:13.488 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:12:14.232 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:12:14.233 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:12:14.014122Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:12:14.014122Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:12:14.014122Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:12:14.014122Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:12:14.014122Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:12:14.014122Z", "running": 0, "size": 1}}] 2022-03-08T19:12:24.259 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:12:24.954 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:12:24.954 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:12:24.765660Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:12:24.765660Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:12:24.765660Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:12:24.765660Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:12:24.765660Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:12:24.765660Z", "running": 0, "size": 1}}] 2022-03-08T19:12:34.987 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:12:35.746 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:12:35.746 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:12:35.514877Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:12:35.514877Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:12:35.514877Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:12:35.514877Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:12:35.514877Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:12:35.514877Z", "running": 0, "size": 1}}] 2022-03-08T19:12:45.767 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:12:46.531 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:12:46.531 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:12:46.331211Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:12:46.331211Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:12:46.331211Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:12:46.331211Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:12:46.331211Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:12:46.331211Z", "running": 0, "size": 1}}] 2022-03-08T19:12:56.556 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:12:57.255 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:12:57.255 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:12:57.084314Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:12:57.084314Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:12:57.084314Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:12:57.084314Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:12:57.084314Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:12:57.084314Z", "running": 0, "size": 1}}] 2022-03-08T19:13:07.282 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:13:08.027 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:13:08.028 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:13:07.851521Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:13:07.851521Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:13:07.851521Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:13:07.851521Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:13:07.851521Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:13:07.851521Z", "running": 0, "size": 1}}] 2022-03-08T19:13:18.053 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:13:18.791 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:13:18.792 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:13:18.604325Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:13:18.604325Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:13:18.604325Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:13:18.604325Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:13:18.604325Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:13:18.604325Z", "running": 0, "size": 1}}] 2022-03-08T19:13:28.818 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:13:29.532 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:13:29.532 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:13:29.346231Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:13:29.346231Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:13:29.346231Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:13:29.346231Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:13:29.346231Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:13:29.346231Z", "running": 0, "size": 1}}] 2022-03-08T19:13:39.558 DEBUG:teuthology.orchestra.run.smithi002:> kubectl -n rook-ceph exec rook-ceph-tools-7677f7f4b9-6d9st -- ceph orch ls -f json 2022-03-08T19:13:40.275 INFO:teuthology.orchestra.run.smithi002.stdout: 2022-03-08T19:13:40.275 INFO:teuthology.orchestra.run.smithi002.stdout:[{"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:41.000000Z", "last_refresh": "2022-03-08T19:13:40.100569Z", "running": 3, "size": 3}}, {"placement": {"count": 1}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:38:56.000000Z", "last_refresh": "2022-03-08T19:13:40.100569Z", "running": 1, "size": 1}}, {"placement": {"count": 3}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:36:58.000000Z", "last_refresh": "2022-03-08T19:13:40.100569Z", "running": 3, "size": 3}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8a96e9d18eedf3630c065f424f31a2567b7edb14aa925c8e107fd263e395a32b", "container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "created": "2022-03-08T18:40:39.000000Z", "last_refresh": "2022-03-08T19:13:40.100569Z", "running": 24, "size": 12}, "unmanaged": true}, {"placement": {"host_pattern": "*"}, "service_id": "all-available-devices", "service_name": "osd.all-available-devices", "service_type": "osd", "spec": {"data_devices": {"all": true}, "filter_logic": "AND", "objectstore": "bluestore"}, "status": {"last_refresh": "2022-03-08T19:13:40.100569Z", "running": 0, "size": 0}}, {"placement": {"count": 1}, "service_id": "foo", "service_name": "rgw.foo", "service_type": "rgw", "spec": {"rgw_frontend_port": 80}, "status": {"container_image_name": "quay.ceph.io/ceph-ci/ceph:6fe8e680a334072b1ba19b2de006862f98aedbae", "last_refresh": "2022-03-08T19:13:40.100569Z", "running": 0, "size": 1}}] 2022-03-08T19:13:40.298 ERROR:teuthology.contextutil:Saw exception from nested tasks Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_teuthology_eeaeee38264fad8a01fd1ce912d8908676ed64bc/teuthology/contextutil.py", line 33, in nested yield vars File "/home/teuthworker/src/github.com_ceph_ceph_b8fc4097cbbda97d59dca39eb86991e7f313985b/qa/tasks/rook.py", line 669, in task while proceed(): File "/home/teuthworker/src/git.ceph.com_git_teuthology_eeaeee38264fad8a01fd1ce912d8908676ed64bc/teuthology/contextutil.py", line 133, in __call__ raise MaxWhileTries(error_msg) teuthology.exceptions.MaxWhileTries: 'waiting for service removal' reached maximum tries (90) after waiting for 900 seconds 2022-03-08T19:13:40.299 ERROR:tasks.rook:'waiting for service removal' reached maximum tries (90) after waiting for 900 seconds Traceback (most recent call last): File "/home/teuthworker/src/github.com_ceph_ceph_b8fc4097cbbda97d59dca39eb86991e7f313985b/qa/tasks/rook.py", line 530, in ceph_config_keyring yield File "/home/teuthworker/src/git.ceph.com_git_teuthology_eeaeee38264fad8a01fd1ce912d8908676ed64bc/teuthology/contextutil.py", line 33, in nested yield vars File "/home/teuthworker/src/github.com_ceph_ceph_b8fc4097cbbda97d59dca39eb86991e7f313985b/qa/tasks/rook.py", line 669, in task while proceed(): File "/home/teuthworker/src/git.ceph.com_git_teuthology_eeaeee38264fad8a01fd1ce912d8908676ed64bc/teuthology/contextutil.py", line 133, in __call__ raise MaxWhileTries(error_msg) teuthology.exceptions.MaxWhileTries: 'waiting for service removal' reached maximum tries (90) after waiting for 900 seconds 2022-03-08T19:13:40.300 INFO:tasks.rook:Cleaning up config and client.admin keyring 2022-03-08T19:13:40.300 DEBUG:teuthology.orchestra.run.smithi002:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2022-03-08T19:13:40.315 DEBUG:teuthology.orchestra.run.smithi066:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2022-03-08T19:13:40.330 DEBUG:teuthology.orchestra.run.smithi137:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2022-03-08T19:13:40.345 ERROR:tasks.rook:'waiting for service removal' reached maximum tries (90) after waiting for 900 seconds Traceback (most recent call last): File "/home/teuthworker/src/github.com_ceph_ceph_b8fc4097cbbda97d59dca39eb86991e7f313985b/qa/tasks/rook.py", line 478, in rook_post_config yield File "/home/teuthworker/src/git.ceph.com_git_teuthology_eeaeee38264fad8a01fd1ce912d8908676ed64bc/teuthology/contextutil.py", line 33, in nested yield vars File "/home/teuthworker/src/github.com_ceph_ceph_b8fc4097cbbda97d59dca39eb86991e7f313985b/qa/tasks/rook.py", line 669, in task while proceed(): File "/home/teuthworker/src/git.ceph.com_git_teuthology_eeaeee38264fad8a01fd1ce912d8908676ed64bc/teuthology/contextutil.py", line 133, in __call__ raise MaxWhileTries(error_msg) teuthology.exceptions.MaxWhileTries: 'waiting for service removal' reached maximum tries (90) after waiting for 900 seconds 2022-03-08T19:13:40.346 ERROR:tasks.rook:'waiting for service removal' reached maximum tries (90) after waiting for 900 seconds Traceback (most recent call last): File "/home/teuthworker/src/github.com_ceph_ceph_b8fc4097cbbda97d59dca39eb86991e7f313985b/qa/tasks/rook.py", line 442, in rook_toolbox yield File "/home/teuthworker/src/git.ceph.com_git_teuthology_eeaeee38264fad8a01fd1ce912d8908676ed64bc/teuthology/contextutil.py", line 33, in nested yield vars File "/home/teuthworker/src/github.com_ceph_ceph_b8fc4097cbbda97d59dca39eb86991e7f313985b/qa/tasks/rook.py", line 669, in task while proceed(): File "/home/teuthworker/src/git.ceph.com_git_teuthology_eeaeee38264fad8a01fd1ce912d8908676ed64bc/teuthology/contextutil.py", line 133, in __call__ raise MaxWhileTries(error_msg) teuthology.exceptions.MaxWhileTries: 'waiting for service removal' reached maximum tries (90) after waiting for 900 seconds 2022-03-08T19:13:40.384 DEBUG:teuthology.orchestra.remote:smithi002:rook/cluster/examples/kubernetes/ceph/operator.yaml is 22KB 2022-03-08T19:13:40.400 DEBUG:teuthology.orchestra.run.smithi002:> kubectl delete -f rook/cluster/examples/kubernetes/ceph/toolbox.yaml 2022-03-08T19:13:40.462 INFO:teuthology.orchestra.run.smithi002.stdout:deployment.apps "rook-ceph-tools" deleted 2022-03-08T19:13:40.480 ERROR:tasks.rook:'waiting for service removal' reached maximum tries (90) after waiting for 900 seconds Traceback (most recent call last): File "/home/teuthworker/src/github.com_ceph_ceph_b8fc4097cbbda97d59dca39eb86991e7f313985b/qa/tasks/rook.py", line 379, in rook_cluster yield File "/home/teuthworker/src/git.ceph.com_git_teuthology_eeaeee38264fad8a01fd1ce912d8908676ed64bc/teuthology/contextutil.py", line 33, in nested yield vars File "/home/teuthworker/src/github.com_ceph_ceph_b8fc4097cbbda97d59dca39eb86991e7f313985b/qa/tasks/rook.py", line 669, in task while proceed(): File "/home/teuthworker/src/git.ceph.com_git_teuthology_eeaeee38264fad8a01fd1ce912d8908676ed64bc/teuthology/contextutil.py", line 133, in __call__ raise MaxWhileTries(error_msg) teuthology.exceptions.MaxWhileTries: 'waiting for service removal' reached maximum tries (90) after waiting for 900 seconds 2022-03-08T19:13:40.481 DEBUG:teuthology.orchestra.run.smithi002:> kubectl delete -f cluster.yaml 2022-03-08T19:13:40.579 INFO:teuthology.orchestra.run.smithi002.stdout:cephcluster.ceph.rook.io "rook-ceph" deleted 2022-03-08T19:13:40.580 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:13:40.581874 I | ceph-cluster-controller: CR "rook-ceph" is going be deleted, cancelling any ongoing orchestration 2022-03-08T19:13:40.877 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:13:40.879388 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:13:40.892 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:13:40.894124 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:13:40.892 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:13:40.894151 I | op-k8sutil: Reporting Event rook-ceph:rook-ceph Warning:ReconcileFailed:CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:13:51.114 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:13:51.115980 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:13:51.144 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:13:51.146483 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:14:01.550 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:14:01.551612 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:14:01.569 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:14:01.570638 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:14:11.791 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:14:11.792745 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:14:11.808 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:14:11.809963 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:14:22.037 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:14:22.038303 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:14:22.060 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:14:22.061989 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:14:32.284 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:14:32.286025 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:14:32.297 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:14:32.298929 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:14:42.526 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:14:42.527511 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:14:42.542 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:14:42.544409 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:14:52.767 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:14:52.769137 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:14:52.784 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:14:52.785936 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:15:03.010 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:15:03.011601 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:15:03.026 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:15:03.026221 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:15:13.270 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:15:13.272135 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:15:13.287 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:15:13.289093 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:15:23.510 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:15:23.511872 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:15:23.523 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:15:23.525304 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:15:33.945 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:15:33.946770 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:15:33.979 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:15:33.980347 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:15:44.208 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:15:44.209158 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:15:44.225 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:15:44.226403 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:15:54.456 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:15:54.457817 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:15:54.470 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:15:54.471840 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:16:04.695 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:16:04.696279 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:16:04.756 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:16:04.755618 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:16:14.973 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:16:14.974897 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:16:14.989 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:16:14.989766 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:16:25.206 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:16:25.207679 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:16:25.269 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:16:25.270845 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:16:35.507 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:16:35.508345 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:16:35.520 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:16:35.521622 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:16:45.745 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:16:45.746479 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:16:45.759 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:16:45.760704 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:16:55.983 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:16:55.984462 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:16:55.998 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:16:56.000613 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:17:06.224 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:17:06.225820 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:17:06.238 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:17:06.240095 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:17:16.486 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:17:16.487689 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:17:16.501 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:17:16.502888 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:17:26.776 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:17:26.777542 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:17:26.792 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:17:26.793961 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:17:37.011 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:17:37.012370 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:17:37.019 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:17:37.021713 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:17:47.276 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:17:47.277336 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:17:47.292 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:17:47.294391 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:17:57.515 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:17:57.516699 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:17:57.528 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:17:57.530789 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:18:07.753 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:07.754517 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:18:07.766 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:07.768638 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:18:14.928 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:14.929584 I | op-mon: parsing mon endpoints: b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 2022-03-08T19:18:15.594 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:15.595388 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T19:18:15.583+0000 7f95a1ca0340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T19:18:15.595 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:18:15.583+0000 7f95a1ca0340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T19:18:15.595 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:18:15.583+0000 7f95a1ca0340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T19:18:15.595 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T19:18:15.596 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:15.595417 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T19:18:15.583+0000 7f95a1ca0340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T19:18:15.596 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:18:15.583+0000 7f95a1ca0340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T19:18:15.597 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:18:15.583+0000 7f95a1ca0340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T19:18:15.597 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T19:18:18.041 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:18.043429 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:18:18.058 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:18.060122 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:18:28.284 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:28.286333 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:18:28.302 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:28.303754 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:18:38.529 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:38.530663 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:18:38.543 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:38.545632 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:18:48.768 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:48.770303 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:18:48.780 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:48.781667 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:18:59.003 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:59.004284 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:18:59.019 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:18:59.020661 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:19:09.241 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:19:09.243536 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:19:09.256 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:19:09.258152 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:19:19.479 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:19:19.480778 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:19:19.494 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:19:19.496220 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:19:29.723 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:19:29.724396 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:19:29.743 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:19:29.743749 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:19:40.014 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:19:40.015633 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:19:40.028 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:19:40.030271 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:19:50.249 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:19:50.250480 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:19:50.261 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:19:50.263033 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:20:00.483 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:20:00.484407 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:20:00.498 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:20:00.500228 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:20:10.722 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:20:10.722808 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:20:10.780 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:20:10.782354 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:20:21.044 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:20:21.045700 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:20:21.060 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:20:21.062604 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:20:31.306 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:20:31.307653 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:20:31.318 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:20:31.320159 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:20:41.546 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:20:41.547125 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:20:41.559 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:20:41.561557 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:20:51.781 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:20:51.782546 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:20:51.798 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:20:51.800167 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:21:02.023 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:21:02.024879 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:21:02.051 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:21:02.053645 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:21:12.274 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:21:12.275293 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:21:12.290 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:21:12.292132 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:21:22.518 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:21:22.519637 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:21:22.535 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:21:22.537199 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:21:32.759 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:21:32.761030 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:21:32.775 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:21:32.776097 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:21:42.996 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:21:42.997795 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:21:43.009 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:21:43.011027 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:21:53.246 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:21:53.247874 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:21:53.270 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:21:53.271920 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:22:03.546 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:22:03.542951 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:22:03.560 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:22:03.562390 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:22:13.782 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:22:13.783834 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:22:13.800 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:22:13.801971 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:22:24.029 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:22:24.031062 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:22:24.073 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:22:24.071030 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:22:34.298 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:22:34.299671 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:22:34.311 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:22:34.313452 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:22:44.539 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:22:44.541068 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:22:44.553 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:22:44.555091 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:22:54.778 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:22:54.779154 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:22:54.792 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:22:54.794226 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:23:05.032 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:23:05.033520 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:23:05.049 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:23:05.051599 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:23:15.271 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:23:15.272782 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:23:15.283 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:23:15.285340 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:23:25.504 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:23:25.505905 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:23:25.518 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:23:25.519670 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:23:35.742 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:23:35.743672 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:23:35.755 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:23:35.756952 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:23:45.981 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:23:45.982848 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:23:45.992 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:23:45.994165 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:23:56.240 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:23:56.242061 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:23:56.255 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:23:56.257273 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:24:06.506 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:24:06.507422 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:24:06.522 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:24:06.523974 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:24:16.753 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:24:16.754080 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:24:16.766 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:24:16.767461 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:24:26.986 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:24:26.986981 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:24:26.999 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:24:27.001180 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:24:37.223 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:24:37.224450 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:24:37.243 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:24:37.244985 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:24:47.470 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:24:47.471416 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:24:47.486 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:24:47.488010 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:24:57.721 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:24:57.722864 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:24:57.734 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:24:57.735973 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:25:07.957 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:25:07.958148 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:25:07.973 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:25:07.974591 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:25:18.195 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:25:18.196547 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:25:18.214 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:25:18.215452 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:25:28.448 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:25:28.449583 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:25:28.462 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:25:28.463934 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:25:38.692 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:25:38.692981 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:25:38.709 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:25:38.710338 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:25:48.930 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:25:48.931887 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:25:50.144 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:25:48.946255 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:25:59.166 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:25:59.167430 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:25:59.176 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:25:59.177357 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:25:59.287 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:25:59.288688 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 19:25:59.276089421 +0000 UTC m=+3043.545631910 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T19:26:09.403 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:26:09.404665 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:26:09.419 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:26:09.420734 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:26:19.642 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:26:19.643390 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:26:19.656 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:26:19.657898 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:26:29.880 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:26:29.881681 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:26:29.894 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:26:29.895380 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:26:40.119 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:26:40.119696 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:26:40.134 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:26:40.135302 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:26:50.361 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:26:50.362414 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:26:50.376 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:26:50.377500 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:27:00.600 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:27:00.601604 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:27:00.616 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:27:00.617991 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:27:10.919 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:27:10.920413 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:27:10.932 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:27:10.933174 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:27:21.153 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:27:21.153788 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:27:21.172 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:27:21.173591 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:27:31.456 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:27:31.456423 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:27:31.470 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:27:31.471811 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:27:41.695 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:27:41.695996 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:27:41.710 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:27:41.711847 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:27:51.932 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:27:51.933222 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:27:51.950 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:27:51.950970 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:28:02.172 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:28:02.172660 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:28:02.187 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:28:02.188208 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:28:12.404 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:28:12.404815 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:28:12.449 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:28:12.450329 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:28:22.669 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:28:22.670572 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:28:22.725 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:28:22.726468 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:28:32.947 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:28:32.948465 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:28:32.964 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:28:32.965819 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:28:43.251 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:28:43.252083 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:28:43.267 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:28:43.268665 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:28:53.508 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:28:53.508909 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:28:53.522 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:28:53.523472 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:29:03.739 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:29:03.740088 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:29:03.753 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:29:03.754128 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:29:13.981 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:29:13.982029 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:29:13.993 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:29:13.994676 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:29:24.215 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:29:24.215753 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:29:24.233 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:29:24.234348 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:29:34.454 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:29:34.454992 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:29:34.471 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:29:34.471949 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:29:44.703 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:29:44.704123 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:29:44.715 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:29:44.716181 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:29:54.944 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:29:54.944676 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:29:54.961 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:29:54.961963 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:30:05.182 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:30:05.182884 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:30:05.200 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:30:05.201038 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:30:15.425 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:30:15.425643 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:30:15.440 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:30:15.440465 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:30:25.703 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:30:25.703805 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:30:25.720 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:30:25.721247 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:30:35.940 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:30:35.940852 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:30:35.958 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:30:35.958967 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:30:46.180 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:30:46.180678 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:30:46.811 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:30:46.197829 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:30:56.418 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:30:56.418703 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:30:56.434 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:30:56.435405 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:31:06.656 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:31:06.656686 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:31:06.674 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:31:06.675272 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:31:16.896 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:31:16.896919 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:31:16.908 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:31:16.909799 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:31:27.131 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:31:27.131360 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:31:27.144 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:31:27.145058 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:31:37.368 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:31:37.368746 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:31:37.389 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:31:37.389784 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:31:47.616 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:31:47.616904 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:31:47.630 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:31:47.631026 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:31:57.913 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:31:57.913661 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:31:57.928 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:31:57.929583 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:32:08.165 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:32:08.165404 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:32:08.175 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:32:08.175950 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:32:18.399 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:32:18.400456 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:32:18.416 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:32:18.417380 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:32:28.639 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:32:28.639758 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:32:28.652 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:32:28.653897 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:32:38.879 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:32:38.880175 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:32:38.889 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:32:38.890169 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:32:49.108 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:32:49.108745 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:32:49.125 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:32:49.124027 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:32:59.346 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:32:59.346605 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:32:59.361 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:32:59.362502 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:33:09.588 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:33:09.589045 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:33:09.604 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:33:09.605018 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:33:19.836 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:33:19.837127 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:33:19.852 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:33:19.852735 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:33:30.074 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:33:30.074368 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:33:30.088 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:33:30.088848 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:33:40.333 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:33:40.333885 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:33:40.348 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:33:40.347621 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:33:50.577 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:33:50.578170 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:33:50.600 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:33:50.601032 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:34:00.822 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:00.823543 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:34:00.843 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:00.844368 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:34:11.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:11.088114 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:34:11.100 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:11.101502 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:34:21.336 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:21.336642 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:34:21.353 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:21.353404 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:34:31.578 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:31.578417 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:34:31.593 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:31.593998 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:34:41.825 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:41.824965 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:34:41.835 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:41.836635 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:34:52.092 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:52.092583 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:34:52.106 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:52.107103 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:34:55.600 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:55.601016 I | op-mon: parsing mon endpoints: b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 2022-03-08T19:34:56.350 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:56.350574 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T19:34:56.342+0000 7f664009d340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T19:34:56.351 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:34:56.342+0000 7f664009d340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T19:34:56.351 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:34:56.342+0000 7f664009d340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T19:34:56.351 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T19:34:56.352 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:34:56.350607 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T19:34:56.342+0000 7f664009d340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T19:34:56.352 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:34:56.342+0000 7f664009d340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T19:34:56.352 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:34:56.342+0000 7f664009d340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T19:34:56.352 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T19:35:02.329 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:35:02.330023 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:35:02.345 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:35:02.346145 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:35:12.569 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:35:12.569822 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:35:12.584 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:35:12.585036 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:35:22.807 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:35:22.807502 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:35:22.822 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:35:22.823530 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:35:33.075 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:35:33.075600 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:35:33.091 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:35:33.091973 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:35:43.326 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:35:43.326571 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:35:43.340 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:35:43.341551 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:35:53.558 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:35:53.558929 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:35:53.572 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:35:53.573268 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:36:03.798 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:36:03.798831 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:36:03.808 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:36:03.809171 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:36:14.029 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:36:14.029877 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:36:14.044 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:36:14.045214 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:36:24.267 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:36:24.267774 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:36:24.280 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:36:24.281410 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:36:34.501 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:36:34.502242 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:36:34.517 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:36:34.516715 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:36:44.756 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:36:44.756254 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:36:44.769 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:36:44.769488 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:36:55.047 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:36:55.048411 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:36:55.081 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:36:55.082444 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:37:05.303 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:37:05.303992 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:37:05.320 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:37:05.320981 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:37:15.548 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:37:15.549488 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:37:15.563 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:37:15.564426 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:37:25.786 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:37:25.787045 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:37:25.803 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:37:25.803876 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:37:36.041 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:37:36.042067 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:37:36.094 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:37:36.094721 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:37:46.337 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:37:46.338554 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:37:46.378 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:37:46.379179 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:37:56.618 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:37:56.613838 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:37:56.624 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:37:56.625603 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:38:06.854 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:38:06.854876 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:38:06.869 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:38:06.869850 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:38:17.144 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:38:17.145407 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:38:17.156 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:38:17.157456 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:38:27.423 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:38:27.423847 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:38:27.456 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:38:27.457179 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:38:37.683 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:38:37.680944 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:38:37.696 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:38:37.696837 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:38:47.973 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:38:47.974748 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:38:47.991 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:38:47.990835 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:38:58.213 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:38:58.213997 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:38:58.225 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:38:58.226257 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:39:08.450 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:39:08.450549 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:39:08.465 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:39:08.466044 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:39:18.686 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:39:18.687483 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:39:18.725 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:39:18.726055 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:39:28.982 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:39:28.978264 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:39:29.038 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:39:29.029835 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:39:39.297 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:39:39.293692 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:39:39.308 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:39:39.309786 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:39:49.529 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:39:49.529102 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:39:49.548 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:39:49.546278 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:39:59.771 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:39:59.771182 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:39:59.782 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:39:59.782976 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:40:10.004 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:40:10.004822 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:40:10.019 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:40:10.020496 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:40:20.267 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:40:20.267540 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:40:20.282 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:40:20.282891 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:40:30.502 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:40:30.502385 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:40:30.516 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:40:30.517440 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:40:40.767 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:40:40.767948 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:40:40.780 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:40:40.781480 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:40:50.999 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:40:50.999652 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:40:51.016 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:40:51.012753 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:41:01.232 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:41:01.232923 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:41:01.245 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:41:01.245927 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:41:11.471 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:41:11.472178 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:41:11.489 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:41:11.489949 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:41:21.727 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:41:21.726900 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:41:21.741 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:41:21.741927 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:41:31.988 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:41:31.988795 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:41:32.003 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:41:32.004239 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:41:42.225 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:41:42.225730 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:41:42.236 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:41:42.236879 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:41:52.459 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:41:52.459567 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:41:52.474 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:41:52.474688 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:42:02.757 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:42:02.748315 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:42:02.761 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:42:02.761811 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:42:12.980 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:42:12.981629 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:42:12.996 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:42:12.997422 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:42:13.369 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:42:13.369990 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 19:42:13.35774899 +0000 UTC m=+4017.627291491 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T19:42:23.218 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:42:23.219727 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:42:23.233 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:42:23.234368 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:42:33.455 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:42:33.456456 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:42:33.468 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:42:33.470000 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:42:43.701 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:42:43.701798 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:42:43.712 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:42:43.713965 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:42:53.960 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:42:53.961781 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:42:53.977 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:42:53.978833 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:43:04.254 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:43:04.255140 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:43:04.277 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:43:04.278468 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:43:14.496 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:43:14.496923 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:43:14.512 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:43:14.513550 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:43:24.735 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:43:24.735988 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:43:24.748 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:43:24.749899 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:43:34.972 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:43:34.973650 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:43:34.992 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:43:34.993274 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:43:45.234 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:43:45.235884 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:43:45.245 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:43:45.245973 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:43:55.465 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:43:55.464865 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:43:55.500 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:43:55.476988 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:44:05.697 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:44:05.698605 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:44:05.709 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:44:05.711072 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:44:15.927 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:44:15.928144 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:44:15.940 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:44:15.941562 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:44:26.183 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:44:26.184377 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:44:26.200 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:44:26.201806 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:44:36.424 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:44:36.425961 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:44:36.440 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:44:36.442037 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:44:46.668 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:44:46.668432 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:44:46.680 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:44:46.682517 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:44:56.920 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:44:56.921313 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:44:56.930 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:44:56.932565 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:45:07.156 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:45:07.157441 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:45:07.170 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:45:07.172667 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:45:17.391 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:45:17.392235 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:45:17.405 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:45:17.407424 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:45:27.631 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:45:27.632934 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:45:27.650 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:45:27.652505 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:45:37.868 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:45:37.869714 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:45:37.883 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:45:37.885728 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:45:48.112 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:45:48.113577 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:45:48.129 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:45:48.131417 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:45:58.350 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:45:58.352063 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:45:58.367 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:45:58.368753 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:46:08.590 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:46:08.591611 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:46:08.607 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:46:08.609387 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:46:18.849 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:46:18.851140 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:46:18.861 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:46:18.862756 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:46:29.081 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:46:29.083170 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:46:29.101 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:46:29.099426 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:46:39.319 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:46:39.320506 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:46:39.333 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:46:39.335372 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:46:49.556 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:46:49.558474 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:46:49.572 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:46:49.573746 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:46:59.808 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:46:59.810576 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:46:59.821 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:46:59.823561 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:47:10.042 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:47:10.044306 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:47:10.057 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:47:10.059799 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:47:20.297 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:47:20.299012 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:47:20.316 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:47:20.318101 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:47:30.577 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:47:30.578854 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:47:30.594 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:47:30.596121 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:47:40.834 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:47:40.836568 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:47:40.851 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:47:40.853485 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:47:51.078 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:47:51.080593 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:47:51.095 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:47:51.097695 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:48:01.349 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:48:01.351468 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:48:01.367 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:48:01.369882 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:48:11.639 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:48:11.641726 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:48:11.655 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:48:11.657630 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:48:21.946 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:48:21.948093 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:48:21.962 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:48:21.964796 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:48:32.191 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:48:32.193058 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:48:32.205 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:48:32.207824 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:48:42.430 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:48:42.432093 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:48:42.449 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:48:42.451519 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:48:52.670 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:48:52.671520 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:48:52.684 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:48:52.686451 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:49:02.909 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:49:02.910294 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:49:02.923 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:49:02.925630 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:49:13.150 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:49:13.151924 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:49:13.212 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:49:13.212605 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:49:23.431 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:49:23.433401 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:49:23.447 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:49:23.449615 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:49:33.688 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:49:33.690431 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:49:33.698 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:49:33.701142 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:49:43.921 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:49:43.922888 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:49:43.939 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:49:43.941735 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:49:54.157 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:49:54.159456 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:49:54.170 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:49:54.172964 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:50:04.399 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:50:04.400616 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:50:04.412 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:50:04.415239 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:50:14.632 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:50:14.634091 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:50:14.651 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:50:14.653861 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:50:24.941 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:50:24.943514 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:50:24.957 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:50:24.958831 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:50:35.182 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:50:35.184200 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:50:35.196 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:50:35.198800 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:50:45.420 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:50:45.422279 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:50:45.434 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:50:45.436595 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:50:55.659 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:50:55.659998 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:50:55.673 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:50:55.674922 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:51:05.895 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:05.897118 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:51:05.914 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:05.916465 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:51:16.137 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:16.139299 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:51:16.153 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:16.155915 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:51:26.375 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:26.377591 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:51:26.389 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:26.392332 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:51:36.356 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:36.358377 I | op-mon: parsing mon endpoints: b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 2022-03-08T19:51:36.649 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:36.651590 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:51:36.663 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:36.665876 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:51:37.085 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:37.087518 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T19:51:37.076+0000 7f3f9bec2340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T19:51:37.086 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:51:37.076+0000 7f3f9bec2340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T19:51:37.086 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:51:37.076+0000 7f3f9bec2340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T19:51:37.086 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T19:51:37.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:37.087549 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T19:51:37.076+0000 7f3f9bec2340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T19:51:37.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:51:37.076+0000 7f3f9bec2340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T19:51:37.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T19:51:37.076+0000 7f3f9bec2340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T19:51:37.087 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T19:51:46.937 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:46.939758 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:51:46.959 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:46.959361 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:51:57.180 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:57.182296 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:51:57.189 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:51:57.192429 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:52:07.411 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:52:07.413937 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:52:07.422 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:52:07.425464 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:52:17.648 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:52:17.650507 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:52:17.660 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:52:17.663450 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:52:27.884 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:52:27.886888 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:52:27.900 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:52:27.902813 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:52:38.124 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:52:38.126967 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:52:38.139 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:52:38.141783 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:52:48.362 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:52:48.365286 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:52:48.382 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:52:48.379143 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:52:58.604 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:52:58.605293 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:52:58.616 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:52:58.618757 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:53:08.840 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:53:08.843039 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:53:08.855 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:53:08.857700 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:53:19.079 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:53:19.081492 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:53:19.091 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:53:19.094546 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:53:29.320 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:53:29.322820 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:53:29.334 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:53:29.337037 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:53:39.571 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:53:39.574439 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:53:39.598 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:53:39.601253 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:53:49.828 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:53:49.830631 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:53:49.840 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:53:49.842886 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:54:00.069 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:54:00.071578 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:54:00.079 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:54:00.082690 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:54:10.300 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:54:10.302793 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:54:10.316 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:54:10.319240 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:54:20.569 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:54:20.571864 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:54:20.583 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:54:20.585997 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:54:30.807 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:54:30.810067 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:54:30.824 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:54:30.827445 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:54:41.058 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:54:41.060526 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:54:41.075 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:54:41.078536 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:54:51.306 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:54:51.308713 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:54:51.322 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:54:51.325621 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:55:01.548 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:55:01.550742 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:55:01.563 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:55:01.566879 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:55:11.788 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:55:11.791066 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:55:11.855 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:55:11.858323 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:55:22.140 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:55:22.143611 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:55:22.177 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:55:22.180319 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:55:32.470 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:55:32.473332 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:55:32.492 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:55:32.492173 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:55:42.712 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:55:42.714749 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:55:42.726 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:55:42.729644 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:55:52.975 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:55:52.978162 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:55:52.989 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:55:52.992365 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:56:03.209 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:56:03.212551 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:56:03.224 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:56:03.227059 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:56:13.443 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:56:13.446527 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:56:13.458 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:56:13.461034 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:56:23.683 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:56:23.686250 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:56:23.700 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:56:23.703723 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:56:34.031 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:56:34.034638 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:56:34.046 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:56:34.049459 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:56:44.265 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:56:44.268556 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:56:44.283 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:56:44.286995 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:56:54.505 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:56:54.508623 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:56:54.520 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:56:54.522430 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:57:04.772 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:57:04.775471 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:57:04.786 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:57:04.789140 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:57:15.005 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:57:15.007921 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:57:15.018 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:57:15.021850 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:57:25.261 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:57:25.264285 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:57:25.277 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:57:25.280836 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:57:35.497 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:57:35.500039 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:57:35.511 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:57:35.514662 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:57:45.779 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:57:45.782478 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:57:45.792 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:57:45.795612 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:57:56.015 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:57:56.018237 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:57:56.032 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:57:56.036187 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:58:06.282 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:58:06.285286 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:58:06.297 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:58:06.300447 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:58:16.519 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:58:16.522190 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:58:16.535 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:58:16.538673 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:58:26.759 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:58:26.762445 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:58:26.772 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:58:26.776123 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:58:27.182 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:58:27.185617 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 19:58:27.17535763 +0000 UTC m=+4991.444900142 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T19:58:37.017 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:58:37.019990 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:58:37.031 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:58:37.034999 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:58:47.288 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:58:47.291339 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:58:47.304 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:58:47.307593 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:58:57.544 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:58:57.547875 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:58:57.559 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:58:57.562658 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:59:07.780 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:59:07.783756 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:59:07.798 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:59:07.801784 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:59:18.027 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:59:18.029938 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:59:18.037 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:59:18.040903 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:59:28.267 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:59:28.270798 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:59:28.283 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:59:28.287045 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:59:38.508 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:59:38.511367 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:59:38.530 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:59:38.528628 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:59:48.744 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:59:48.747242 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:59:48.760 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:59:48.763585 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:59:59.012 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:59:59.015682 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T19:59:59.032 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 19:59:59.035220 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:00:09.250 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:00:09.253904 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:00:09.265 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:00:09.268318 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:00:19.493 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:00:19.496123 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:00:19.503 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:00:19.507427 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:00:29.723 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:00:29.726759 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:00:29.737 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:00:29.740614 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:00:39.961 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:00:39.964455 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:00:39.977 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:00:39.980435 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:00:50.200 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:00:50.204100 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:00:50.217 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:00:50.219112 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:01:00.441 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:01:00.444528 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:01:00.459 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:01:00.462996 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:01:10.682 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:01:10.685318 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:01:10.695 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:01:10.699398 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:01:20.916 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:01:20.919831 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:01:20.932 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:01:20.936470 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:01:31.154 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:01:31.157917 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:01:31.168 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:01:31.172124 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:01:41.388 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:01:41.392088 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:01:41.403 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:01:41.406772 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:01:51.624 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:01:51.627896 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:01:51.640 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:01:51.643917 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:02:01.862 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:02:01.866204 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:02:01.903 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:02:01.907011 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:02:12.128 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:02:12.131647 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:02:12.143 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:02:12.147863 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:02:22.381 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:02:22.384779 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:02:22.396 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:02:22.400206 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:02:32.689 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:02:32.693101 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:02:32.704 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:02:32.708658 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:02:42.962 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:02:42.965385 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:02:42.990 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:02:42.993631 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:02:53.211 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:02:53.214863 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:02:53.231 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:02:53.235613 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:03:03.453 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:03:03.456544 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:03:03.466 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:03:03.470375 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:03:13.689 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:03:13.693115 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:03:13.702 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:03:13.704550 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:03:23.923 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:03:23.926874 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:03:23.939 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:03:23.943423 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:03:34.162 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:03:34.165539 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:03:34.176 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:03:34.180589 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:03:44.401 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:03:44.404332 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:03:44.416 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:03:44.419655 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:03:54.640 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:03:54.644136 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:03:54.664 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:03:54.668435 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:04:04.889 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:04:04.892881 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:04:04.906 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:04:04.909143 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:04:15.122 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:04:15.125717 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:04:15.137 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:04:15.141606 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:04:25.367 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:04:25.370796 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:04:25.381 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:04:25.385490 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:04:35.603 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:04:35.607012 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:04:35.705 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:04:35.619645 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:04:45.837 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:04:45.840401 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:04:45.855 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:04:45.859141 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:04:56.081 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:04:56.085211 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:04:56.096 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:04:56.100132 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:05:06.331 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:05:06.334807 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:05:06.346 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:05:06.350468 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:05:16.571 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:05:16.574566 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:05:16.598 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:05:16.601121 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:05:26.820 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:05:26.824221 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:05:26.833 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:05:26.837402 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:05:37.055 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:05:37.058887 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:05:37.072 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:05:37.076028 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:05:47.300 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:05:47.303839 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:05:47.320 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:05:47.321760 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:05:57.565 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:05:57.568017 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:05:57.577 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:05:57.581397 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:06:07.838 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:06:07.842032 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:06:07.853 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:06:07.857590 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:06:18.121 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:06:18.124997 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:06:18.150 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:06:18.153998 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:06:28.396 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:06:28.400098 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:06:28.408 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:06:28.412342 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:06:38.675 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:06:38.678731 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:06:38.686 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:06:38.690432 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:06:48.906 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:06:48.910250 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:06:49.007 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:06:49.009284 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:06:59.230 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:06:59.233691 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:06:59.262 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:06:59.265856 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:07:09.520 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:07:09.524449 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:07:09.534 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:07:09.538026 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:07:19.757 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:07:19.761342 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:07:19.772 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:07:19.775974 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:07:29.999 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:07:30.001864 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:07:30.016 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:07:30.020403 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:07:40.236 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:07:40.239800 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:07:40.249 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:07:40.253206 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:07:50.469 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:07:50.472581 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:07:50.485 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:07:50.488557 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:08:00.706 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:00.709745 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:08:00.720 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:00.724605 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:08:10.943 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:10.947330 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:08:10.958 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:10.962145 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:08:17.091 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:17.094797 I | op-mon: parsing mon endpoints: b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 2022-03-08T20:08:17.725 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:17.727738 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T20:08:17.715+0000 7f3a5c825340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T20:08:18.562 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:08:17.715+0000 7f3a5c825340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T20:08:18.563 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:08:17.715+0000 7f3a5c825340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T20:08:18.563 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T20:08:18.563 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:17.727763 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T20:08:17.715+0000 7f3a5c825340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T20:08:18.564 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:08:17.715+0000 7f3a5c825340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T20:08:18.564 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:08:17.715+0000 7f3a5c825340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T20:08:18.565 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T20:08:21.181 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:21.184928 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:08:21.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:21.196855 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:08:31.415 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:31.418072 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:08:31.431 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:31.434607 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:08:41.693 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:41.697009 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:08:41.712 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:41.716294 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:08:51.956 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:51.959094 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:08:51.971 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:08:51.974601 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:09:02.192 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:09:02.195954 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:09:02.203 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:09:02.206689 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:09:12.438 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:09:12.441605 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:09:12.452 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:09:12.456349 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:09:22.680 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:09:22.683864 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:09:22.695 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:09:22.699695 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:09:32.919 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:09:32.922795 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:09:32.932 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:09:32.936126 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:09:43.156 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:09:43.160112 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:09:43.170 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:09:43.173768 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:09:53.388 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:09:53.392205 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:09:53.405 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:09:53.408836 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:10:03.626 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:10:03.629265 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:10:03.640 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:10:03.644152 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:10:13.866 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:10:13.869554 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:10:13.879 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:10:13.883509 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:10:24.103 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:10:24.106566 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:10:24.118 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:10:24.121651 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:10:34.346 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:10:34.350081 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:10:34.362 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:10:34.365774 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:10:44.585 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:10:44.588692 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:10:44.622 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:10:44.625521 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:10:54.880 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:10:54.884115 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:10:54.898 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:10:54.901708 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:11:05.125 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:11:05.128587 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:11:05.138 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:11:05.141850 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:11:15.359 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:11:15.362817 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:11:15.371 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:11:15.374791 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:11:25.594 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:11:25.597995 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:11:25.609 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:11:25.613027 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:11:35.826 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:11:35.830277 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:11:35.850 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:11:35.853632 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:11:46.069 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:11:46.072476 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:11:46.082 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:11:46.086275 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:11:56.302 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:11:56.305464 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:11:56.319 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:11:56.321135 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:12:06.541 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:12:06.545063 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:12:06.555 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:12:06.559550 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:12:16.832 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:12:16.836035 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:12:16.842 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:12:16.846327 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:12:27.076 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:12:27.078351 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:12:27.089 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:12:27.092291 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:12:37.313 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:12:37.316594 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:12:37.329 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:12:37.332522 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:12:47.552 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:12:47.555223 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:12:47.566 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:12:47.569985 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:12:57.788 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:12:57.791254 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:12:57.803 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:12:57.806578 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:13:08.024 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:13:08.027940 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:13:08.037 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:13:08.040758 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:13:18.276 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:13:18.279564 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:13:18.292 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:13:18.295808 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:13:28.544 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:13:28.548285 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:13:28.562 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:13:28.563238 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:13:38.781 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:13:38.785314 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:13:38.798 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:13:38.802133 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:13:49.025 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:13:49.028850 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:13:49.040 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:13:49.044562 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:13:49.041 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:13:49.044586 I | op-k8sutil: Reporting Event rook-ceph:rook-ceph Warning:ReconcileFailed:CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:13:59.263 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:13:59.266801 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:13:59.277 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:13:59.281434 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:14:09.497 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:14:09.501272 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:14:09.515 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:14:09.519016 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:14:19.737 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:14:19.740800 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:14:19.751 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:14:19.754893 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:14:29.978 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:14:29.981666 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:14:29.995 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:14:29.998181 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:14:40.225 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:14:40.228871 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:14:40.239 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:14:40.242925 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:14:50.465 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:14:50.468184 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:14:50.511 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:14:50.514935 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:15:00.733 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:15:00.736931 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:15:00.750 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:15:00.753598 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:15:11.009 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:15:11.012912 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:15:11.026 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:15:11.029713 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:15:21.250 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:15:21.253422 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:15:21.279 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:15:21.283114 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:15:31.503 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:15:31.506698 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:15:31.519 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:15:31.523073 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:15:41.737 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:15:41.741109 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:15:41.753 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:15:41.757617 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:15:42.157 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:15:42.160132 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 20:15:42.147451514 +0000 UTC m=+6026.416994019 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T20:15:51.985 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:15:51.988477 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:15:52.000 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:15:52.000927 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:16:02.226 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:16:02.229462 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:16:02.242 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:16:02.245487 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:16:12.464 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:16:12.467614 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:16:12.478 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:16:12.482344 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:16:22.699 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:16:22.702199 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:16:22.717 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:16:22.720818 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:16:32.941 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:16:32.944714 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:16:32.959 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:16:32.963017 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:16:43.016 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:16:43.011049 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 20:16:43.001230435 +0000 UTC m=+6087.270772933 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T20:16:43.184 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:16:43.187354 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:16:43.203 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:16:43.203539 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:16:53.438 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:16:53.441857 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:16:53.451 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:16:53.454831 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:17:03.671 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:17:03.674061 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:17:03.687 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:17:03.690510 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:17:13.910 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:17:13.913681 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:17:13.926 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:17:13.924888 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:17:24.155 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:17:24.158558 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:17:24.165 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:17:24.168890 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:17:34.386 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:17:34.387810 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:17:34.407 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:17:34.409873 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:17:44.639 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:17:44.641873 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:17:44.652 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:17:44.656263 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:17:54.876 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:17:54.879808 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:17:54.891 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:17:54.894569 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:18:05.147 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:18:05.150959 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:18:05.159 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:18:05.162990 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:18:15.388 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:18:15.390818 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:18:15.401 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:18:15.404257 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:18:25.624 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:18:25.627298 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:18:25.705 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:18:25.703353 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:18:35.920 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:18:35.923593 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:18:35.973 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:18:35.976589 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:18:46.196 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:18:46.199889 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:18:46.210 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:18:46.213539 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:18:56.434 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:18:56.437194 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:18:56.450 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:18:56.451883 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:19:06.684 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:19:06.687333 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:19:06.701 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:19:06.704031 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:19:16.923 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:19:16.926208 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:19:16.937 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:19:16.940079 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:19:27.154 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:19:27.157253 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:19:27.192 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:19:27.195213 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:19:37.416 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:19:37.419335 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:19:37.430 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:19:37.433764 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:19:47.652 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:19:47.655641 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:19:47.668 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:19:47.671471 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:19:57.896 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:19:57.899016 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:19:57.911 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:19:57.914299 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:20:08.138 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:20:08.141035 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:20:08.154 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:20:08.157069 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:20:18.385 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:20:18.388514 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:20:18.394 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:20:18.398068 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:20:28.620 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:20:28.623809 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:20:28.638 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:20:28.639879 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:20:38.879 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:20:38.882365 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:20:38.894 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:20:38.898290 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:20:49.117 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:20:49.120138 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:20:49.130 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:20:49.133415 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:20:59.416 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:20:59.419521 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:20:59.435 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:20:59.438301 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:21:09.674 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:21:09.676777 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:21:09.690 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:21:09.693469 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:21:19.914 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:21:19.917442 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:21:19.963 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:21:19.966509 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:21:30.186 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:21:30.189332 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:21:30.201 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:21:30.204679 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:21:40.426 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:21:40.429039 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:21:40.443 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:21:40.446420 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:21:50.662 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:21:50.665150 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:21:50.733 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:21:50.736086 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:22:00.958 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:22:00.960602 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:22:00.989 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:22:00.992481 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:22:11.212 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:22:11.215406 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:22:11.223 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:22:11.226137 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:22:21.446 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:22:21.448959 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:22:21.458 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:22:21.461136 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:22:31.679 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:22:31.682020 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:22:31.696 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:22:31.697946 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:22:41.913 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:22:41.915803 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:22:41.926 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:22:41.930005 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:22:52.157 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:22:52.160389 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:22:52.172 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:22:52.176011 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:23:02.395 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:23:02.398336 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:23:02.414 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:23:02.416552 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:23:12.638 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:23:12.641650 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:23:12.654 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:23:12.657661 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:23:22.938 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:23:22.941306 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:23:22.952 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:23:22.955427 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:23:33.172 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:23:33.174926 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:23:33.190 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:23:33.193487 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:23:43.412 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:23:43.415426 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:23:43.425 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:23:43.428250 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:23:53.647 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:23:53.650800 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:23:53.714 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:23:53.668212 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:24:03.892 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:03.891441 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:24:03.913 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:03.910476 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:24:14.130 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:14.133754 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:24:14.147 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:14.150254 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:24:24.369 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:24.371948 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:24:24.385 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:24.388126 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:24:34.607 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:34.609792 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:24:34.622 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:34.625522 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:24:44.843 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:44.845877 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:24:44.861 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:44.862833 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:24:55.081 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:55.084032 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:24:55.097 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:55.100555 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:24:57.733 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:57.736105 I | op-mon: parsing mon endpoints: b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 2022-03-08T20:24:58.391 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:58.393013 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T20:24:58.381+0000 7ff0eeb2e340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T20:24:58.392 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:24:58.381+0000 7ff0eeb2e340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T20:24:58.392 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:24:58.381+0000 7ff0eeb2e340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T20:24:58.392 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T20:24:58.393 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:24:58.393065 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T20:24:58.381+0000 7ff0eeb2e340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T20:24:58.393 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:24:58.381+0000 7ff0eeb2e340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T20:24:58.393 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:24:58.381+0000 7ff0eeb2e340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T20:24:58.393 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T20:25:05.388 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:25:05.390886 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:25:05.403 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:25:05.406326 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:25:15.625 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:25:15.627716 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:25:15.638 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:25:15.641261 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:25:25.860 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:25:25.863485 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:25:25.874 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:25:25.877825 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:25:36.098 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:25:36.101514 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:25:36.116 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:25:36.119074 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:25:46.343 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:25:46.346043 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:25:46.367 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:25:46.366375 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:25:56.586 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:25:56.589534 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:25:56.648 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:25:56.651570 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:26:06.877 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:26:06.879792 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:26:06.895 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:26:06.899010 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:26:17.118 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:26:17.120851 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:26:17.132 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:26:17.134983 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:26:27.352 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:26:27.355071 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:26:27.367 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:26:27.370768 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:26:37.589 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:26:37.592200 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:26:37.602 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:26:37.606065 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:26:47.820 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:26:47.822521 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:26:47.835 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:26:47.838102 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:26:58.061 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:26:58.063623 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:26:58.075 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:26:58.078345 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:27:08.298 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:27:08.301530 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:27:08.315 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:27:08.318562 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:27:18.543 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:27:18.546227 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:27:18.554 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:27:18.557843 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:27:28.780 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:27:28.783267 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:27:28.794 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:27:28.797587 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:27:39.019 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:27:39.022413 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:27:39.033 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:27:39.036602 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:27:49.258 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:27:49.260798 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:27:49.273 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:27:49.276055 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:27:59.501 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:27:59.498462 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:27:59.511 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:27:59.514064 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:28:09.728 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:28:09.731725 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:28:09.748 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:28:09.749229 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:28:19.974 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:28:19.977048 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:28:19.988 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:28:19.991815 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:28:30.211 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:28:30.214725 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:28:30.233 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:28:30.236300 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:28:40.458 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:28:40.461618 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:28:40.473 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:28:40.476794 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:28:50.697 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:28:50.700787 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:28:50.712 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:28:50.715235 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:29:00.937 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:29:00.940640 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:29:00.954 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:29:00.955848 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:29:11.178 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:29:11.181322 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:29:11.229 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:29:11.232816 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:29:21.450 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:29:21.453159 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:29:21.464 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:29:21.467921 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:29:31.696 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:29:31.699478 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:29:31.710 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:29:31.713895 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:29:41.930 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:29:41.932822 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:29:41.944 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:29:41.947315 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:29:52.201 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:29:52.204273 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:29:52.215 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:29:52.219267 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:30:02.434 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:30:02.437381 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:30:02.453 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:30:02.452922 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:30:12.666 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:30:12.669367 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:30:12.682 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:30:12.685386 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:30:22.904 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:30:22.907035 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:30:22.918 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:30:22.922141 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:30:33.145 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:30:33.148707 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:30:33.157 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:30:33.161190 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:30:43.410 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:30:43.413171 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:30:43.425 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:30:43.428639 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:30:53.651 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:30:53.653912 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:30:53.663 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:30:53.667018 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:31:03.889 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:31:03.892437 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:31:03.903 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:31:03.906930 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:31:14.122 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:31:14.125765 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:31:14.139 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:31:14.142947 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:31:24.365 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:31:24.368911 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:31:24.380 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:31:24.384333 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:31:34.609 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:31:34.612349 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:31:34.622 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:31:34.625303 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:31:44.849 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:31:44.852787 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:31:44.866 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:31:44.869534 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:31:55.115 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:31:55.117946 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:31:55.132 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:31:55.135696 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:32:05.381 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:32:05.384435 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:32:05.393 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:32:05.397162 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:32:15.614 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:32:15.617746 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:32:15.628 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:32:15.631368 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:32:25.852 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:32:25.855773 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:32:25.865 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:32:25.869018 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:32:36.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:32:36.090193 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:32:36.104 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:32:36.107392 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:32:46.375 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:32:46.378250 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:32:46.388 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:32:46.391474 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:32:56.612 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:32:56.614780 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:32:56.625 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:32:56.628814 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:33:06.848 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:33:06.851271 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:33:06.863 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:33:06.867022 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:33:17.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:33:17.090194 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:33:17.099 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:33:17.102814 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:33:27.322 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:33:27.325250 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:33:27.336 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:33:27.339429 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:33:37.592 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:33:37.595509 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:33:37.622 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:33:37.626397 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:33:47.840 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:33:47.843504 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:33:47.859 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:33:47.861559 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:33:58.059 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:33:58.061873 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 20:33:58.023533743 +0000 UTC m=+7122.293076209 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T20:33:58.083 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:33:58.086675 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:33:58.098 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:33:58.102233 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:34:08.322 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:34:08.325397 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:34:08.345 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:34:08.342186 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:34:18.565 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:34:18.569025 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:34:18.578 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:34:18.581897 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:34:28.806 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:34:28.809740 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:34:28.823 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:34:28.825793 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:34:39.046 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:34:39.049345 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:34:39.060 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:34:39.063872 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:34:49.283 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:34:49.286686 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:34:49.301 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:34:49.304300 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:34:59.523 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:34:59.526950 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:34:59.536 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:34:59.540210 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:35:09.763 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:35:09.766234 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:35:09.779 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:35:09.783131 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:35:20.011 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:35:20.014670 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:35:20.033 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:35:20.032515 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:35:30.259 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:35:30.261554 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:35:30.274 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:35:30.277597 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:35:40.497 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:35:40.500490 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:35:40.512 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:35:40.516395 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:35:50.763 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:35:50.766387 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:35:51.223 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:35:50.777921 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:36:01.003 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:36:01.006556 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:36:01.015 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:36:01.018742 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:36:11.282 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:36:11.285836 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:36:11.294 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:36:11.298578 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:36:21.517 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:36:21.520569 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:36:21.532 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:36:21.536206 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:36:31.760 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:36:31.763674 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:36:31.781 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:36:31.784816 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:36:42.008 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:36:42.011641 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:36:42.024 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:36:42.027504 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:36:52.247 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:36:52.250046 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:36:52.262 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:36:52.265895 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:37:02.486 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:37:02.489264 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:37:02.499 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:37:02.503483 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:37:12.722 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:37:12.726193 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:37:12.738 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:37:12.741720 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:37:22.961 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:37:22.964276 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:37:22.975 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:37:22.979058 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:37:33.196 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:37:33.199722 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:37:33.211 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:37:33.215060 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:37:43.439 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:37:43.442471 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:37:43.459 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:37:43.461421 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:37:53.681 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:37:53.684912 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:37:53.722 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:37:53.725789 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:38:03.939 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:38:03.942893 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:38:03.951 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:38:03.955487 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:38:14.172 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:38:14.175603 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:38:14.241 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:38:14.244265 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:38:24.465 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:38:24.468839 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:38:24.482 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:38:24.486249 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:38:34.705 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:38:34.709265 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:38:34.720 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:38:34.724020 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:38:45.019 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:38:45.023166 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:38:45.030 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:38:45.034859 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:38:55.259 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:38:55.263101 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:38:55.277 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:38:55.281067 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:39:05.498 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:39:05.501605 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:39:05.513 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:39:05.516804 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:39:15.743 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:39:15.746522 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:39:15.757 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:39:15.761229 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:39:25.978 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:39:25.982309 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:39:25.994 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:39:25.997670 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:39:36.228 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:39:36.231726 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:39:36.240 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:39:36.244707 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:39:46.471 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:39:46.474381 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:39:46.490 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:39:46.492270 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:39:56.708 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:39:56.711966 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:39:56.749 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:39:56.752694 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:40:06.987 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:40:06.991388 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:40:07.002 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:40:07.006204 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:40:17.227 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:40:17.230692 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:40:17.238 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:40:17.241908 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:40:27.473 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:40:27.476482 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:40:27.486 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:40:27.490426 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:40:37.712 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:40:37.716378 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:40:37.730 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:40:37.734459 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:40:47.957 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:40:47.961038 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:40:47.973 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:40:47.977296 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:40:58.192 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:40:58.196254 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:40:58.215 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:40:58.212776 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:41:08.433 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:08.437510 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:41:08.451 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:08.455908 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:41:18.678 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:18.682282 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:41:18.692 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:18.696189 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:41:28.913 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:28.917108 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:41:28.926 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:28.930360 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:41:38.403 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:38.407099 I | op-mon: parsing mon endpoints: b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 2022-03-08T20:41:39.046 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:39.049799 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T20:41:39.041+0000 7f7e3b32c340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T20:41:39.047 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:41:39.041+0000 7f7e3b32c340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T20:41:39.047 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:41:39.041+0000 7f7e3b32c340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T20:41:39.047 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T20:41:39.048 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:39.049822 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T20:41:39.041+0000 7f7e3b32c340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T20:41:39.048 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:41:39.041+0000 7f7e3b32c340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T20:41:39.048 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:41:39.041+0000 7f7e3b32c340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T20:41:39.048 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T20:41:39.150 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:39.154381 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:41:39.167 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:39.171390 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:41:49.391 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:49.394659 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:41:49.410 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:49.411623 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:41:59.627 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:59.631477 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:41:59.642 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:41:59.646203 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:42:09.867 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:42:09.871362 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:42:09.884 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:42:09.887896 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:42:20.109 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:42:20.112719 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:42:20.123 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:42:20.127385 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:42:30.368 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:42:30.371547 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:42:30.384 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:42:30.388466 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:42:40.611 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:42:40.615121 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:42:40.625 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:42:40.629917 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:42:50.848 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:42:50.851930 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:42:50.859 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:42:50.863274 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:43:01.081 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:43:01.085293 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:43:01.095 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:43:01.098919 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:43:11.324 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:43:11.328458 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:43:11.341 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:43:11.343000 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:43:21.567 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:43:21.570896 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:43:21.582 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:43:21.586559 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:43:31.817 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:43:31.820448 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:43:31.835 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:43:31.839005 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:43:42.059 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:43:42.062836 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:43:42.074 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:43:42.078589 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:43:52.303 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:43:52.306577 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:43:52.318 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:43:52.322668 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:44:02.541 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:44:02.544369 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:44:02.558 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:44:02.560072 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:44:12.781 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:44:12.784647 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:44:12.799 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:44:12.803145 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:44:23.023 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:44:23.026619 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:44:23.037 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:44:23.041403 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:44:33.290 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:44:33.293764 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:44:33.305 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:44:33.309706 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:44:43.572 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:44:43.575691 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:44:43.586 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:44:43.590469 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:44:53.849 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:44:53.853488 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:44:53.866 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:44:53.870213 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:45:04.097 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:45:04.100807 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:45:04.106 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:45:04.110868 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:45:14.330 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:45:14.333848 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:45:14.343 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:45:14.347669 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:45:24.570 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:45:24.573450 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:45:24.591 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:45:24.591062 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:45:34.809 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:45:34.812357 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:45:34.823 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:45:34.826867 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:45:45.042 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:45:45.046129 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:45:45.053 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:45:45.057473 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:45:55.275 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:45:55.279333 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:45:55.292 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:45:55.295992 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:46:05.516 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:46:05.520256 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:46:05.530 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:46:05.534809 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:46:15.774 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:46:15.778263 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:46:15.788 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:46:15.792286 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:46:26.024 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:46:26.028322 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:46:26.039 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:46:26.042995 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:46:36.260 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:46:36.264066 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:46:36.327 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:46:36.331100 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:46:46.548 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:46:46.551953 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:46:46.603 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:46:46.607801 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:46:56.822 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:46:56.826311 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:46:56.837 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:46:56.841032 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:47:07.074 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:47:07.070535 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:47:07.083 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:47:07.087473 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:47:17.308 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:47:17.311470 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:47:17.323 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:47:17.327397 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:47:27.547 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:47:27.550455 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:47:27.561 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:47:27.565279 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:47:37.781 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:47:37.784442 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:47:37.808 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:47:37.803101 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:47:48.041 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:47:48.044470 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:47:48.056 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:47:48.060306 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:47:58.278 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:47:58.281854 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:47:58.294 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:47:58.297923 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:48:08.517 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:48:08.521032 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:48:08.532 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:48:08.536068 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:48:18.755 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:48:18.758930 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:48:18.773 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:48:18.777181 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:48:28.999 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:48:29.002662 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:48:29.013 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:48:29.017234 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:48:39.237 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:48:39.241082 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:48:39.254 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:48:39.257738 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:48:49.478 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:48:49.481870 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:48:49.537 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:48:49.541163 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:48:59.762 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:48:59.765753 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:48:59.777 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:48:59.781202 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:49:10.005 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:49:10.008530 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:49:10.018 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:49:10.022720 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:49:20.283 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:49:20.287221 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:49:20.298 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:49:20.302119 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:49:30.524 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:49:30.527508 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:49:30.540 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:49:30.543826 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:49:40.823 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:49:40.826819 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:49:40.838 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:49:40.841718 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:49:51.062 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:49:51.065952 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:49:51.078 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:49:51.081924 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:50:01.314 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:50:01.318040 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:50:01.332 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:50:01.335525 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:50:11.555 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:50:11.558091 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:50:11.579 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:50:11.575068 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:50:11.984 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:50:11.988078 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 20:50:11.974158792 +0000 UTC m=+8096.243701270 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T20:50:21.799 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:50:21.802648 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:50:21.841 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:50:21.844978 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:50:32.065 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:50:32.068621 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:50:32.081 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:50:32.085034 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:50:42.320 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:50:42.320097 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:50:42.331 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:50:42.334792 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:50:52.558 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:50:52.561718 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:50:52.570 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:50:52.574396 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:51:02.835 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:51:02.838237 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:51:02.851 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:51:02.854805 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:51:13.094 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:51:13.097542 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:51:13.113 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:51:13.115099 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:51:23.337 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:51:23.340903 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:51:23.350 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:51:23.354351 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:51:33.576 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:51:33.578995 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:51:33.591 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:51:33.594557 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:51:43.822 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:51:43.825636 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:51:43.840 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:51:43.843489 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:51:54.065 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:51:54.068738 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:51:54.081 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:51:54.084215 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:52:04.312 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:52:04.315155 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:52:04.328 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:52:04.331793 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:52:14.548 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:52:14.551711 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:52:14.561 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:52:14.565131 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:52:24.786 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:52:24.789187 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:52:24.801 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:52:24.805288 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:52:35.026 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:52:35.029834 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:52:35.037 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:52:35.041008 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:52:45.253 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:52:45.256490 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:52:45.308 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:52:45.311749 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:52:55.572 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:52:55.575800 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:52:55.582 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:52:55.585914 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:53:05.804 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:53:05.807352 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:53:05.818 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:53:05.821987 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:53:16.043 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:53:16.045967 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:53:16.056 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:53:16.059350 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:53:26.319 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:53:26.321968 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:53:26.380 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:53:26.383240 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:53:36.602 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:53:36.605147 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:53:36.625 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:53:36.628841 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:53:46.851 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:53:46.854606 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:53:46.871 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:53:46.874345 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:53:57.095 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:53:57.098220 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:53:57.122 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:53:57.125007 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:54:07.343 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:54:07.346629 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:54:07.358 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:54:07.361986 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:54:17.581 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:54:17.584123 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:54:17.595 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:54:17.599109 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:54:27.824 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:54:27.827509 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:54:27.835 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:54:27.838599 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:54:38.072 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:54:38.075314 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:54:38.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:54:38.090998 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:54:48.371 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:54:48.374570 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:54:48.385 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:54:48.388399 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:54:58.605 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:54:58.608493 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:54:58.618 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:54:58.621621 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:55:08.836 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:55:08.838874 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:55:08.850 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:55:08.853929 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:55:19.083 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:55:19.086589 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:55:19.098 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:55:19.101358 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:55:29.322 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:55:29.324948 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:55:29.336 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:55:29.339316 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:55:39.558 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:55:39.561618 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:55:39.576 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:55:39.579938 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:55:49.799 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:55:49.802653 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:55:49.819 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:55:49.822785 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:56:00.042 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:56:00.045299 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:56:00.057 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:56:00.060430 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:56:10.283 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:56:10.285921 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:56:10.297 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:56:10.300227 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:56:20.521 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:56:20.524373 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:56:20.538 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:56:20.542178 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:56:30.756 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:56:30.758817 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:56:30.801 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:56:30.803908 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:56:41.025 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:56:41.027927 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:56:41.039 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:56:41.042663 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:56:51.259 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:56:51.262437 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:56:51.275 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:56:51.278230 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:57:01.524 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:57:01.527751 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:57:01.543 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:57:01.544860 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:57:11.773 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:57:11.776726 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:57:11.804 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:57:11.806955 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:57:22.023 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:57:22.025837 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:57:22.035 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:57:22.039046 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:57:32.259 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:57:32.262242 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:57:32.274 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:57:32.277156 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:57:42.495 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:57:42.498490 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:57:42.512 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:57:42.515566 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:57:52.776 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:57:52.778519 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:57:52.788 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:57:52.791764 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:58:03.014 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:03.016412 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:58:03.028 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:03.030953 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:58:13.252 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:13.254424 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:58:13.266 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:13.269563 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:58:19.054 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:19.056851 I | op-mon: parsing mon endpoints: b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 2022-03-08T20:58:19.786 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:19.788076 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T20:58:19.779+0000 7f6b51238340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T20:58:19.786 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:58:19.779+0000 7f6b51238340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T20:58:19.786 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:58:19.779+0000 7f6b51238340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T20:58:19.787 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T20:58:19.787 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:19.788111 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T20:58:19.779+0000 7f6b51238340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T20:58:19.787 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:58:19.779+0000 7f6b51238340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T20:58:19.787 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T20:58:19.779+0000 7f6b51238340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T20:58:19.788 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T20:58:23.490 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:23.493302 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:58:23.503 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:23.506526 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:58:33.727 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:33.729758 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:58:33.743 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:33.746558 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:58:43.970 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:43.972855 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:58:43.989 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:43.992681 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:58:54.215 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:54.217886 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:58:54.225 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:58:54.228057 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:59:04.473 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:59:04.476423 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:59:04.489 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:59:04.492994 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:59:14.725 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:59:14.728037 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:59:14.736 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:59:14.739737 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:59:24.964 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:59:24.967290 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:59:24.979 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:59:24.982482 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:59:35.208 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:59:35.210485 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:59:35.223 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:59:35.226946 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:59:45.446 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:59:45.448751 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:59:45.460 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:59:45.463283 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:59:55.685 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:59:55.687898 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T20:59:55.700 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 20:59:55.703177 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:00:05.921 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:00:05.923763 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:00:05.936 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:00:05.939643 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:00:16.159 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:00:16.161990 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:00:16.178 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:00:16.181189 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:00:26.406 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:00:26.409415 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:00:26.423 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:00:26.426581 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:00:36.658 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:00:36.661481 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:00:36.690 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:00:36.693458 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:00:46.915 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:00:46.917985 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:00:46.931 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:00:46.934386 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:00:57.174 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:00:57.176725 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:00:57.183 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:00:57.186595 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:01:07.421 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:01:07.423267 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:01:07.435 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:01:07.438472 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:01:17.702 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:01:17.705122 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:01:17.714 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:01:17.717605 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:01:27.956 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:01:27.959046 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:01:27.968 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:01:27.971791 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:01:38.194 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:01:38.197296 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:01:38.212 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:01:38.215701 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:01:48.445 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:01:48.447676 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:01:48.461 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:01:48.464461 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:01:58.684 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:01:58.687120 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:01:58.699 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:01:58.702830 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:02:08.924 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:02:08.926929 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:02:08.935 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:02:08.938567 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:02:19.162 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:02:19.164753 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:02:19.182 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:02:19.185231 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:02:29.408 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:02:29.411568 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:02:29.423 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:02:29.426871 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:02:39.651 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:02:39.653762 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:02:39.671 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:02:39.669767 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:02:49.891 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:02:49.894486 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:02:49.905 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:02:49.908324 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:03:00.133 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:03:00.135731 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:03:00.149 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:03:00.152480 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:03:10.372 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:03:10.375385 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:03:10.386 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:03:10.388956 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:03:20.613 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:03:20.615939 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:03:20.624 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:03:20.627673 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:03:30.918 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:03:30.920507 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:03:30.933 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:03:30.935435 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:03:41.159 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:03:41.161798 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:03:41.176 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:03:41.179329 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:03:51.397 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:03:51.399912 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:03:51.414 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:03:51.417577 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:04:01.638 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:04:01.641351 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:04:01.653 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:04:01.656104 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:04:11.881 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:04:11.884130 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:04:11.902 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:04:11.902680 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:04:22.124 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:04:22.126878 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:04:22.140 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:04:22.142728 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:04:32.380 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:04:32.383033 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:04:32.396 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:04:32.399465 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:04:42.621 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:04:42.623332 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:04:42.630 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:04:42.633090 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:04:52.854 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:04:52.856741 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:04:52.872 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:04:52.874848 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:05:03.120 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:05:03.120290 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:05:03.138 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:05:03.140945 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:05:13.360 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:05:13.362884 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:05:13.376 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:05:13.379072 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:05:23.598 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:05:23.601245 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:05:23.612 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:05:23.614954 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:05:33.836 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:05:33.839108 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:05:33.881 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:05:33.884002 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:05:44.100 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:05:44.102910 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:05:44.113 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:05:44.116147 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:05:54.337 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:05:54.340147 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:05:54.352 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:05:54.355559 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:06:04.577 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:06:04.579920 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:06:04.589 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:06:04.592402 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:06:14.814 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:06:14.816860 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:06:14.831 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:06:14.834652 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:06:25.052 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:06:25.054896 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:06:25.066 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:06:25.069180 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:06:35.289 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:06:35.292457 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:06:35.307 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:06:35.309721 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:06:45.549 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:06:45.552013 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:06:45.566 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:06:45.569044 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:06:55.792 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:06:55.795044 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:06:55.816 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:06:55.817294 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:07:06.038 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:07:06.041361 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:07:06.055 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:07:06.058815 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:07:16.279 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:07:16.282069 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:07:16.294 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:07:16.296928 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:07:26.519 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:07:26.521613 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:07:26.532 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:07:26.535316 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:07:26.896 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:07:26.899369 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 21:07:26.88255378 +0000 UTC m=+9131.152096265 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T21:07:36.758 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:07:36.760582 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:07:36.772 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:07:36.774735 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:07:46.997 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:07:46.999564 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:07:47.012 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:07:47.015618 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:07:57.269 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:07:57.272206 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:07:57.282 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:07:57.285396 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:08:07.508 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:08:07.510585 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:08:07.523 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:08:07.526459 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:08:17.749 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:08:17.751114 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:08:17.768 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:08:17.770962 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:08:27.835 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:08:27.837944 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 21:08:27.824493249 +0000 UTC m=+9192.094035807 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T21:08:27.993 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:08:27.996642 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:08:28.004 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:08:28.006962 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:08:38.225 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:08:38.227286 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:08:38.240 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:08:38.243319 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:08:48.458 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:08:48.461086 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:08:48.487 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:08:48.489931 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:08:58.748 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:08:58.750400 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:08:58.780 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:08:58.782684 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:09:09.037 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:09:09.039339 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:09:09.500 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:09:09.054556 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:09:19.271 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:09:19.273776 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:09:19.282 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:09:19.285516 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:09:29.502 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:09:29.504987 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:09:29.543 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:09:29.545820 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:09:39.768 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:09:39.769424 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:09:40.666 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:09:39.787889 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:09:50.007 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:09:50.009928 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:09:50.021 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:09:50.024292 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:10:00.274 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:10:00.276764 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:10:00.289 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:10:00.292427 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:10:10.514 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:10:10.516817 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:10:10.525 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:10:10.528728 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:10:20.754 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:10:20.756120 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:10:20.766 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:10:20.768874 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:10:30.989 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:10:30.992322 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:10:31.005 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:10:31.007754 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:10:41.238 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:10:41.240542 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:10:41.253 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:10:41.255808 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:10:51.477 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:10:51.479775 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:10:51.487 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:10:51.490189 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:11:01.720 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:11:01.722710 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:11:01.734 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:11:01.737280 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:11:11.995 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:11:11.998181 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:11:12.019 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:11:12.016417 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:11:22.238 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:11:22.240234 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:11:22.252 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:11:22.255501 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:11:32.484 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:11:32.486164 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:11:32.500 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:11:32.502457 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:11:42.726 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:11:42.728266 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:11:42.742 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:11:42.745397 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:11:52.980 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:11:52.983243 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:11:52.991 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:11:52.993792 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:12:03.219 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:12:03.221483 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:12:03.230 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:12:03.233441 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:12:13.450 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:12:13.452689 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:12:13.461 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:12:13.463985 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:12:23.683 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:12:23.685521 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:12:23.697 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:12:23.699868 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:12:33.915 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:12:33.917721 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:12:33.931 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:12:33.934601 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:12:44.151 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:12:44.153485 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:12:44.163 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:12:44.166134 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:12:54.386 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:12:54.389271 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:12:54.401 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:12:54.404082 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:13:04.622 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:13:04.625148 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:13:04.634 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:13:04.636654 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:13:14.858 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:13:14.860983 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:13:14.875 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:13:14.878324 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:13:25.124 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:13:25.126722 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:13:25.146 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:13:25.149419 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:13:35.364 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:13:35.367471 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:13:35.375 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:13:35.377866 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:13:45.599 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:13:45.601760 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:13:45.615 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:13:45.617470 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:13:55.841 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:13:55.844007 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:13:55.856 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:13:55.858691 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:13:55.856 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:13:55.858721 I | op-k8sutil: Reporting Event rook-ceph:rook-ceph Warning:ReconcileFailed:CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:14:06.083 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:14:06.085844 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:14:06.101 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:14:06.103654 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:14:16.324 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:14:16.327014 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:14:16.340 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:14:16.343523 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:14:26.611 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:14:26.614051 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:14:26.703 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:14:26.705000 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:14:36.937 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:14:36.939318 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:14:36.949 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:14:36.951737 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:14:47.185 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:14:47.187662 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:14:47.202 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:14:47.204705 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:14:57.428 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:14:57.431125 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:14:57.445 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:14:57.447475 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:14:59.791 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:14:59.793934 I | op-mon: parsing mon endpoints: b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 2022-03-08T21:15:00.476 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:00.478521 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T21:15:00.465+0000 7f4cb6d7c340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T21:15:00.477 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T21:15:00.465+0000 7f4cb6d7c340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T21:15:00.477 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T21:15:00.465+0000 7f4cb6d7c340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T21:15:00.477 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T21:15:00.477 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:00.478565 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T21:15:00.465+0000 7f4cb6d7c340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T21:15:00.478 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T21:15:00.465+0000 7f4cb6d7c340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T21:15:00.478 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T21:15:00.465+0000 7f4cb6d7c340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T21:15:00.478 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T21:15:07.681 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:07.684281 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:15:07.694 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:07.697462 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:15:17.950 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:17.952207 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:15:17.968 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:17.971081 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:15:28.188 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:28.190622 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:15:28.201 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:28.204154 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:15:38.424 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:38.426683 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:15:38.439 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:38.442443 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:15:48.662 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:48.664472 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:15:48.675 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:48.677621 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:15:58.894 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:58.897014 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:15:58.908 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:15:58.910726 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:16:09.130 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:16:09.132241 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:16:09.143 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:16:09.146057 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:16:19.391 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:16:19.393922 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:16:19.402 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:16:19.405248 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:16:29.689 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:16:29.691509 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:16:29.701 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:16:29.703931 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:16:39.944 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:16:39.946453 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:16:39.961 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:16:39.963160 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:16:50.185 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:16:50.188007 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:16:50.196 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:16:50.198970 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:17:00.420 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:17:00.423146 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:17:00.439 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:17:00.442207 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:17:10.665 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:17:10.667553 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:17:10.678 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:17:10.680511 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:17:20.901 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:17:20.903704 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:17:20.915 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:17:20.918308 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:17:31.194 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:17:31.196642 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:17:31.208 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:17:31.210780 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:17:41.435 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:17:41.437644 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:17:41.451 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:17:41.453695 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:17:51.673 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:17:51.676058 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:17:51.686 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:17:51.689324 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:18:01.912 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:18:01.915039 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:18:01.945 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:18:01.947745 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:18:12.177 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:18:12.179502 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:18:12.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:18:12.195812 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:18:22.422 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:18:22.424451 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:18:22.434 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:18:22.436860 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:18:32.667 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:18:32.669525 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:18:32.684 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:18:32.685873 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:18:42.907 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:18:42.909610 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:18:42.929 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:18:42.931711 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:18:53.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:18:53.195209 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:18:53.210 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:18:53.212931 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:19:03.434 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:19:03.436901 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:19:03.453 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:19:03.456507 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:19:13.677 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:19:13.679243 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:19:13.689 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:19:13.692031 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:19:23.910 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:19:23.912255 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:19:23.925 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:19:23.928009 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:19:34.147 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:19:34.149510 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:19:34.161 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:19:34.163650 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:19:44.390 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:19:44.392752 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:19:44.404 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:19:44.407235 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:19:54.625 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:19:54.628037 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:19:54.641 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:19:54.643612 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:20:04.934 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:20:04.936923 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:20:04.950 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:20:04.953204 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:20:15.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:20:15.195127 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:20:15.209 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:20:15.211763 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:20:25.428 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:20:25.430412 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:20:25.443 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:20:25.445606 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:20:35.683 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:20:35.685464 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:20:35.699 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:20:35.702345 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:20:45.923 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:20:45.925983 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:20:45.933 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:20:45.935891 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:20:56.161 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:20:56.164112 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:20:56.175 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:20:56.177791 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:21:06.418 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:21:06.420705 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:21:06.437 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:21:06.433127 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:21:16.681 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:21:16.683764 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:21:16.699 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:21:16.702096 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:21:26.917 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:21:26.919607 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:21:26.932 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:21:26.934851 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:21:37.158 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:21:37.160108 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:21:37.170 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:21:37.172612 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:21:47.396 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:21:47.398821 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:21:47.412 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:21:47.414688 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:21:57.639 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:21:57.641622 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:21:57.656 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:21:57.658844 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:22:07.885 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:22:07.887627 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:22:07.899 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:22:07.902104 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:22:18.141 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:22:18.142922 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:22:18.155 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:22:18.158383 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:22:28.382 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:22:28.384433 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:22:28.398 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:22:28.400453 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:22:38.623 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:22:38.625244 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:22:38.633 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:22:38.635798 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:22:48.857 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:22:48.859709 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:22:48.872 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:22:48.875022 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:22:59.097 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:22:59.099701 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:22:59.110 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:22:59.113376 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:23:09.374 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:23:09.376573 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:23:09.390 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:23:09.392712 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:23:19.619 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:23:19.620867 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:23:19.645 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:23:19.648153 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:23:29.876 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:23:29.878253 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:23:29.890 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:23:29.892946 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:23:40.119 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:23:40.121682 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:23:40.130 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:23:40.133182 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:23:50.353 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:23:50.355248 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:23:50.367 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:23:50.369634 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:24:00.585 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:24:00.587696 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:24:00.601 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:24:00.604578 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:24:10.839 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:24:10.841839 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:24:10.865 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:24:10.862337 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:24:21.085 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:24:21.087563 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:24:21.096 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:24:21.099181 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:24:31.324 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:24:31.326281 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:24:31.341 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:24:31.343914 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:24:41.568 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:24:41.570366 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:24:41.582 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:24:41.584460 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:24:41.925 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:24:41.927337 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 21:24:41.910560697 +0000 UTC m=+10166.180103249 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T21:24:51.823 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:24:51.826139 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:24:51.841 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:24:51.843496 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:25:02.061 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:25:02.063621 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:25:02.074 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:25:02.077043 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:25:12.313 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:25:12.315731 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:25:12.327 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:25:12.330302 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:25:22.548 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:25:22.551146 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:25:22.564 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:25:22.567322 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:25:32.788 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:25:32.790613 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:25:32.800 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:25:32.803240 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:25:42.870 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:25:42.872714 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 21:25:42.862301922 +0000 UTC m=+10227.131844371 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T21:25:43.023 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:25:43.025364 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:25:43.040 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:25:43.042828 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:25:53.261 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:25:53.263379 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:25:53.278 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:25:53.280803 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:26:03.502 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:26:03.504981 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:26:03.515 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:26:03.517846 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:26:13.739 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:26:13.741246 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:26:13.756 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:26:13.759223 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:26:23.980 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:26:23.982265 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:26:23.993 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:26:23.996483 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:26:34.222 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:26:34.225183 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:26:34.233 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:26:34.235932 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:26:44.458 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:26:44.460680 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:26:44.476 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:26:44.478777 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:26:54.700 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:26:54.703136 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:26:54.716 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:26:54.718857 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:27:04.940 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:27:04.942797 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:27:04.955 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:27:04.957766 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:27:15.179 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:27:15.181415 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:27:15.195 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:27:15.197950 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:27:25.422 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:27:25.425178 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:27:25.495 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:27:25.498473 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:27:35.724 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:27:35.726617 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:27:35.755 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:27:35.758195 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:27:45.985 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:27:45.988153 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:27:45.996 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:27:45.999537 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:27:56.217 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:27:56.220233 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:27:56.232 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:27:56.235319 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:28:06.468 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:28:06.470524 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:28:06.481 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:28:06.484193 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:28:16.703 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:28:16.705459 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:28:16.717 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:28:16.720409 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:28:26.991 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:28:26.994265 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:28:27.002 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:28:27.005418 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:28:37.225 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:28:37.227885 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:28:37.240 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:28:37.243658 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:28:47.487 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:28:47.489740 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:28:47.499 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:28:47.502025 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:28:57.718 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:28:57.721336 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:28:57.736 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:28:57.738389 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:29:07.959 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:29:07.962271 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:29:07.975 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:29:07.978108 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:29:18.205 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:29:18.208399 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:29:18.220 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:29:18.223566 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:29:28.446 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:29:28.448460 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:29:28.462 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:29:28.465043 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:29:38.686 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:29:38.688548 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:29:38.831 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:29:38.832911 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:29:49.059 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:29:49.061775 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:29:49.079 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:29:49.077879 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:29:59.299 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:29:59.301202 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:29:59.318 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:29:59.321211 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:30:09.540 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:30:09.542785 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:30:09.554 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:30:09.557096 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:30:19.781 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:30:19.784268 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:30:19.798 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:30:19.801593 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:30:30.023 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:30:30.025976 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:30:30.038 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:30:30.040975 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:30:40.267 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:30:40.269627 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:30:40.283 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:30:40.285911 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:30:50.505 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:30:50.508015 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:30:50.515 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:30:50.518645 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:31:00.739 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:00.741386 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:31:00.752 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:00.755138 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:31:10.979 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:10.978975 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:31:10.992 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:10.994745 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:31:21.214 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:21.217488 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:31:21.230 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:21.233633 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:31:31.457 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:31.460133 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:31:31.470 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:31.473499 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:31:40.483 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:40.486425 I | op-mon: parsing mon endpoints: b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 2022-03-08T21:31:41.124 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:41.126457 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T21:31:41.119+0000 7fbb75bee340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T21:31:41.125 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T21:31:41.119+0000 7fbb75bee340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T21:31:41.125 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T21:31:41.119+0000 7fbb75bee340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T21:31:41.125 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T21:31:41.125 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:41.126491 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T21:31:41.119+0000 7fbb75bee340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T21:31:41.126 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T21:31:41.119+0000 7fbb75bee340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T21:31:41.126 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T21:31:41.119+0000 7fbb75bee340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T21:31:41.126 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T21:31:41.694 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:41.697339 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:31:41.713 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:41.716121 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:31:51.933 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:51.935940 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:31:51.947 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:31:51.950427 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:32:02.199 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:32:02.202259 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:32:02.213 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:32:02.216750 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:32:12.465 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:32:12.467658 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:32:12.483 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:32:12.485754 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:32:22.722 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:32:22.725079 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:32:22.738 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:32:22.741241 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:32:32.958 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:32:32.961042 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:32:32.973 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:32:32.975981 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:32:43.232 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:32:43.234489 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:32:43.247 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:32:43.250088 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:32:53.483 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:32:53.486202 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:32:53.497 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:32:53.499262 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:33:03.722 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:33:03.725245 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:33:03.740 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:33:03.742842 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:33:13.963 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:33:13.965754 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:33:13.982 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:33:13.984928 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:33:24.200 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:33:24.202269 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:33:24.208 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:33:24.211585 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:33:34.432 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:33:34.434463 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:33:34.447 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:33:34.450696 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:33:44.670 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:33:44.672598 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:33:44.682 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:33:44.685568 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:33:54.907 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:33:54.909940 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:33:54.922 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:33:54.924855 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:34:05.177 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:34:05.180105 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:34:05.189 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:34:05.192291 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:34:15.424 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:34:15.426832 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:34:15.438 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:34:15.441132 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:34:25.656 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:34:25.658854 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:34:25.671 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:34:25.673690 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:34:35.893 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:34:35.896001 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:34:35.907 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:34:35.910447 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:34:46.128 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:34:46.130934 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:34:46.142 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:34:46.145245 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:34:56.365 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:34:56.367276 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:34:56.380 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:34:56.383431 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:35:06.611 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:35:06.614330 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:35:06.629 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:35:06.632467 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:35:16.855 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:35:16.857904 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:35:16.874 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:35:16.876825 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:35:27.100 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:35:27.102761 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:35:27.114 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:35:27.117444 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:35:37.339 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:35:37.341923 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:35:37.354 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:35:37.354564 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:35:47.586 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:35:47.588411 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:35:47.649 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:35:47.651613 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:35:57.904 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:35:57.907348 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:35:57.916 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:35:57.918881 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:36:08.135 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:36:08.137471 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:36:08.147 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:36:08.149685 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:36:18.389 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:36:18.391701 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:36:18.403 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:36:18.405678 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:36:28.656 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:36:28.658790 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:36:28.671 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:36:28.674366 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:36:38.896 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:36:38.898781 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:36:38.912 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:36:38.915686 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:36:49.178 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:36:49.180359 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:36:49.192 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:36:49.195553 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:36:59.416 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:36:59.418240 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:36:59.449 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:36:59.451949 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:37:09.673 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:37:09.675757 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:37:09.721 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:37:09.723922 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:37:19.944 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:37:19.946540 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:37:19.959 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:37:19.962339 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:37:30.206 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:37:30.208927 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:37:30.219 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:37:30.222786 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:37:40.444 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:37:40.446158 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:37:40.459 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:37:40.462071 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:37:50.687 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:37:50.689839 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:37:50.702 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:37:50.705047 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:38:00.922 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:38:00.924702 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:38:00.938 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:38:00.940549 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:38:11.159 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:38:11.161470 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:38:11.172 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:38:11.175261 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:38:21.442 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:38:21.444662 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:38:21.457 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:38:21.459667 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:38:31.706 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:38:31.708547 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:38:31.722 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:38:31.724704 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:38:41.943 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:38:41.945771 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:38:41.959 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:38:41.961919 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:38:52.179 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:38:52.181990 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:38:52.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:38:52.196337 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:39:02.418 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:39:02.420697 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:39:02.434 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:39:02.437185 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:39:12.656 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:39:12.659272 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:39:12.670 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:39:12.673392 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:39:22.897 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:39:22.899993 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:39:22.913 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:39:22.916536 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:39:33.163 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:39:33.165750 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:39:33.178 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:39:33.181286 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:39:43.400 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:39:43.402652 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:39:43.471 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:39:43.472872 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:39:53.693 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:39:53.695825 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:39:53.709 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:39:53.712258 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:40:03.935 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:40:03.937092 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:40:03.947 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:40:03.950166 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:40:14.215 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:40:14.217777 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:40:14.229 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:40:14.232082 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:40:24.450 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:40:24.452075 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:40:24.465 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:40:24.468080 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:40:34.692 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:40:34.694579 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:40:34.707 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:40:34.709725 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:40:44.928 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:40:44.930673 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:40:44.944 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:40:44.947487 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:40:55.166 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:40:55.168773 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:40:55.180 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:40:55.183529 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:41:05.409 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:41:05.410842 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:41:05.424 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:41:05.427113 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:41:15.647 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:41:15.649814 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:41:15.661 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:41:15.664256 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:41:25.886 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:41:25.888115 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:41:25.901 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:41:25.903646 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:41:36.126 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:41:36.128783 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:41:36.142 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:41:36.145336 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:41:46.367 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:41:46.370054 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:41:46.380 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:41:46.383598 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:41:56.607 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:41:56.609768 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:41:56.622 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:41:56.625551 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:41:56.943 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:41:56.945643 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 21:41:56.932451683 +0000 UTC m=+11201.201994189 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T21:42:06.840 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:42:06.843311 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:42:06.855 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:42:06.858109 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:42:17.088 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:42:17.090487 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:42:17.102 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:42:17.105025 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:42:27.381 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:42:27.383660 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:42:27.400 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:42:27.402529 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:42:37.626 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:42:37.628454 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:42:37.642 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:42:37.645325 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:42:47.864 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:42:47.866559 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:42:47.881 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:42:47.884034 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:42:58.105 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:42:58.107675 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:42:58.119 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:42:58.122150 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:43:08.345 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:43:08.347496 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:43:08.361 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:43:08.363707 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:43:18.584 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:43:18.586653 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:43:18.604 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:43:18.606957 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:43:28.827 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:43:28.829257 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:43:28.842 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:43:28.844531 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:43:39.087 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:43:39.090259 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:43:39.108 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:43:39.110962 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:43:49.329 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:43:49.331437 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:43:49.341 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:43:49.343733 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:43:59.607 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:43:59.609935 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:43:59.621 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:43:59.624487 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:44:09.851 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:44:09.849615 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:44:09.861 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:44:09.863578 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:44:20.093 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:44:20.095695 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:44:20.109 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:44:20.111563 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:44:30.325 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:44:30.327679 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:44:30.345 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:44:30.348437 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:44:40.574 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:44:40.575938 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:44:40.598 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:44:40.600927 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:44:50.821 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:44:50.823988 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:44:50.840 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:44:50.842259 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:45:01.060 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:45:01.062993 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:45:01.123 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:45:01.126307 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:45:11.349 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:45:11.351634 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:45:11.373 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:45:11.366975 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:45:21.587 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:45:21.588586 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:45:21.640 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:45:21.642482 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:45:31.868 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:45:31.870035 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:45:31.883 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:45:31.885178 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:45:42.124 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:45:42.126875 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:45:42.137 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:45:42.139939 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:45:52.363 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:45:52.364928 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:45:52.378 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:45:52.380555 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:46:02.597 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:46:02.599733 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:46:02.611 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:46:02.613439 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:46:12.837 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:46:12.839404 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:46:12.850 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:46:12.852703 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:46:23.068 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:46:23.070014 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:46:23.081 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:46:23.084063 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:46:33.311 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:46:33.312849 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:46:33.346 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:46:33.348873 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:46:43.598 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:46:43.600292 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:46:43.611 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:46:43.614269 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:46:53.833 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:46:53.834860 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:46:53.884 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:46:53.886757 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:47:04.110 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:47:04.112454 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:47:04.125 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:47:04.127808 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:47:14.348 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:47:14.350632 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:47:14.362 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:47:14.364855 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:47:24.588 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:47:24.590487 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:47:24.604 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:47:24.606285 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:47:34.838 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:47:34.840296 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:47:34.855 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:47:34.857456 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:47:45.112 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:47:45.113682 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:47:45.131 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:47:45.128404 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:47:55.351 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:47:55.352697 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:47:55.366 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:47:55.367639 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:48:05.589 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:05.590632 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:48:05.606 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:05.607997 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:48:15.827 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:15.828739 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:48:15.841 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:15.843250 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:48:21.131 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:21.133523 I | op-mon: parsing mon endpoints: b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 2022-03-08T21:48:21.801 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:21.802222 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T21:48:21.792+0000 7f288b716340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T21:48:21.801 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T21:48:21.792+0000 7f288b716340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T21:48:21.802 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T21:48:21.792+0000 7f288b716340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T21:48:21.802 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T21:48:21.802 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:21.802252 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T21:48:21.792+0000 7f288b716340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T21:48:21.802 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T21:48:21.792+0000 7f288b716340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T21:48:21.803 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T21:48:21.792+0000 7f288b716340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T21:48:21.803 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T21:48:26.071 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:26.072991 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:48:26.091 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:26.089997 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:48:36.323 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:36.325079 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:48:36.339 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:36.341417 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:48:46.567 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:46.569214 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:48:46.621 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:46.623103 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:48:56.842 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:56.843657 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:48:56.856 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:48:56.858576 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:49:07.092 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:49:07.094194 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:49:07.105 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:49:07.107693 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:49:17.334 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:49:17.336507 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:49:17.352 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:49:17.354156 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:49:27.574 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:49:27.576516 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:49:27.590 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:49:27.591994 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:49:37.813 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:49:37.815070 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:49:37.828 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:49:37.830528 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:49:48.053 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:49:48.054581 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:49:48.067 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:49:48.069248 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:49:58.290 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:49:58.292099 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:49:58.304 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:49:58.306556 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:50:08.531 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:50:08.533279 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:50:08.546 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:50:08.548713 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:50:18.770 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:50:18.771425 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:50:18.784 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:50:18.786100 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:50:29.012 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:50:29.013540 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:50:29.026 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:50:29.027858 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:50:39.243 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:50:39.245181 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:50:39.259 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:50:39.261571 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:50:49.508 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:50:49.510432 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:50:49.523 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:50:49.523587 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:50:59.803 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:50:59.804713 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:50:59.817 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:50:59.819469 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:51:10.062 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:51:10.063823 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:51:10.076 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:51:10.078414 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:51:20.312 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:51:20.314257 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:51:20.346 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:51:20.348018 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:51:30.571 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:51:30.572845 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:51:30.587 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:51:30.588938 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:51:40.805 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:51:40.807160 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:51:40.817 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:51:40.819763 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:51:51.042 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:51:51.043490 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:51:51.055 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:51:51.057443 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:52:01.279 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:52:01.281612 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:52:01.293 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:52:01.295205 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:52:11.520 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:52:11.522168 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:52:11.533 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:52:11.535168 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:52:21.756 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:52:21.757931 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:52:21.767 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:52:21.769390 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:52:31.995 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:52:31.996958 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:52:32.011 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:52:32.013183 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:52:42.237 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:52:42.238749 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:52:42.252 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:52:42.254515 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:52:52.497 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:52:52.499288 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:52:52.515 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:52:52.517309 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:53:02.746 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:53:02.747534 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:53:02.758 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:53:02.760105 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:53:12.985 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:53:12.987083 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:53:13.000 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:53:13.002123 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:53:23.260 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:53:23.261416 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:53:23.276 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:53:23.277738 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:53:33.498 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:53:33.500378 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:53:33.517 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:53:33.516839 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:53:43.735 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:53:43.737401 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:53:43.754 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:53:43.756070 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:53:53.980 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:53:53.981538 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:53:53.996 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:53:53.997854 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:54:04.223 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:54:04.224883 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:54:04.242 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:54:04.243688 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:54:14.468 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:54:14.469953 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:54:14.482 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:54:14.484188 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:54:24.712 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:54:24.713838 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:54:24.729 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:54:24.730591 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:54:34.959 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:54:34.960676 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:54:34.975 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:54:34.976666 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:54:45.205 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:54:45.206984 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:54:45.220 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:54:45.221642 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:54:55.487 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:54:55.488472 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:54:55.513 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:54:55.515169 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:55:05.749 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:55:05.750197 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:55:05.764 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:55:05.766127 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:55:15.986 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:55:15.987320 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:55:15.995 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:55:15.997013 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:55:26.216 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:55:26.218086 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:55:26.228 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:55:26.230303 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:55:36.463 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:55:36.465163 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:55:36.477 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:55:36.479281 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:55:46.703 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:55:46.705200 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:55:46.716 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:55:46.717646 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:55:56.936 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:55:56.937651 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:55:56.951 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:55:56.952960 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:56:07.226 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:56:07.227758 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:56:07.243 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:56:07.244629 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:56:17.490 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:56:17.491952 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:56:17.504 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:56:17.505741 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:56:27.724 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:56:27.725985 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:56:27.739 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:56:27.741344 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:56:37.963 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:56:37.964311 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:56:37.977 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:56:37.978529 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:56:48.206 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:56:48.207103 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:56:48.220 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:56:48.222149 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:56:58.442 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:56:58.443634 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:56:58.456 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:56:58.457974 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:57:08.686 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:57:08.687558 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:57:08.739 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:57:08.735816 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:57:18.965 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:57:18.966372 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:57:18.978 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:57:18.979789 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:57:29.206 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:57:29.206843 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:57:29.219 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:57:29.220632 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:57:39.440 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:57:39.441911 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:57:39.450 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:57:39.451960 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:57:49.694 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:57:49.695655 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:57:49.710 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:57:49.711738 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:57:59.932 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:57:59.933754 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:57:59.946 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:57:59.947859 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:58:10.166 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:58:10.166892 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:58:10.185 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:58:10.182598 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:58:20.405 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:58:20.406304 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:58:20.420 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:58:20.421750 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:58:30.644 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:58:30.645982 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:58:30.659 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:58:30.661025 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:58:40.890 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:58:40.891559 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:58:40.909 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:58:40.908699 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:58:51.133 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:58:51.134368 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:58:51.148 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:58:51.150001 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:59:01.366 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:59:01.367053 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:59:01.381 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:59:01.382757 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:59:11.605 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:59:11.606543 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:59:11.622 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:59:11.624147 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:59:11.809 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:59:11.810190 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 21:59:11.802170707 +0000 UTC m=+12236.071713189 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T21:59:21.846 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:59:21.847922 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:59:21.862 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:59:21.863708 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:59:32.105 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:59:32.107209 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:59:32.120 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:59:32.122016 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:59:42.402 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:59:42.403686 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:59:42.419 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:59:42.421022 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:59:52.641 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:59:52.642716 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T21:59:52.656 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 21:59:52.658240 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:00:02.882 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:00:02.883976 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:00:02.900 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:00:02.901279 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:00:13.122 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:00:13.124003 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:00:13.138 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:00:13.139756 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:00:23.358 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:00:23.359303 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:00:23.371 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:00:23.372683 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:00:33.597 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:00:33.598548 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:00:33.613 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:00:33.614853 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:00:43.838 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:00:43.839730 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:00:43.851 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:00:43.852940 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:00:54.072 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:00:54.073298 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:00:54.086 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:00:54.087957 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:01:04.308 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:01:04.309932 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:01:04.323 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:01:04.324916 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:01:14.552 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:01:14.553476 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:01:14.566 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:01:14.568360 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:01:24.796 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:01:24.796940 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:01:24.812 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:01:24.813955 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:01:35.035 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:01:35.036301 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:01:35.046 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:01:35.047676 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:01:45.325 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:01:45.326530 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:01:45.339 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:01:45.340391 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:01:55.588 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:01:55.589623 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:01:55.604 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:01:55.605939 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:02:05.823 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:02:05.824423 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:02:05.874 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:02:05.876277 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:02:16.101 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:02:16.102683 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:02:16.117 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:02:16.119147 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:02:26.350 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:02:26.351058 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:02:26.365 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:02:26.366157 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:02:36.623 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:02:36.624593 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:02:36.634 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:02:36.636356 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:02:46.864 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:02:46.865862 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:02:46.881 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:02:46.880239 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:02:57.120 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:02:57.121704 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:02:57.135 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:02:57.137173 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:03:07.360 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:03:07.361334 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:03:07.375 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:03:07.376716 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:03:17.595 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:03:17.596984 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:03:17.610 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:03:17.611808 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:03:27.882 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:03:27.883279 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:03:27.897 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:03:27.898833 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:03:38.120 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:03:38.121439 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:03:38.138 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:03:38.139697 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:03:48.370 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:03:48.371509 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:03:48.386 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:03:48.387721 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:03:58.607 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:03:58.608332 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:03:58.624 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:03:58.624894 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:04:08.842 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:04:08.843235 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:04:08.855 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:04:08.857304 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:04:19.079 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:04:19.080354 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:04:19.095 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:04:19.096181 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:04:29.319 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:04:29.320999 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:04:29.333 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:04:29.335178 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:04:39.559 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:04:39.560306 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:04:39.574 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:04:39.575753 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:04:49.799 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:04:49.800694 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:04:49.813 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:04:49.814628 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:05:00.034 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:00.035687 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:05:00.049 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:00.050250 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:05:01.810 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:01.811804 I | op-mon: parsing mon endpoints: b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 2022-03-08T22:05:02.515 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:02.515644 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T22:05:02.505+0000 7f4201624340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T22:05:02.515 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T22:05:02.505+0000 7f4201624340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T22:05:02.516 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T22:05:02.505+0000 7f4201624340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T22:05:02.516 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T22:05:02.516 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:02.515678 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T22:05:02.505+0000 7f4201624340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T22:05:02.517 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T22:05:02.505+0000 7f4201624340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T22:05:02.517 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T22:05:02.505+0000 7f4201624340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T22:05:02.517 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T22:05:10.273 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:10.274541 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:05:10.286 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:10.287906 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:05:20.511 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:20.512564 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:05:20.525 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:20.526919 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:05:30.749 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:30.750782 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:05:30.768 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:30.769770 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:05:40.988 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:40.989104 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:05:41.013 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:41.013058 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:05:51.234 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:51.235444 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:05:51.250 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:05:51.251578 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:06:01.477 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:06:01.478339 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:06:01.511 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:06:01.497057 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:06:11.748 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:06:11.749554 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:06:11.759 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:06:11.761329 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:06:21.986 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:06:21.987991 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:06:22.003 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:06:22.004732 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:06:32.230 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:06:32.231736 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:06:32.245 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:06:32.246849 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:06:42.471 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:06:42.472479 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:06:42.490 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:06:42.491450 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:06:52.711 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:06:52.712894 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:06:52.727 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:06:52.729171 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:07:02.957 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:07:02.958463 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:07:02.970 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:07:02.971548 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:07:13.189 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:07:13.190394 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:07:13.254 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:07:13.255716 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:07:23.494 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:07:23.495277 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:07:23.505 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:07:23.506648 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:07:33.763 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:07:33.764380 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:07:33.780 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:07:33.781731 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:07:44.000 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:07:44.001616 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:07:44.010 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:07:44.011691 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:07:54.235 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:07:54.236054 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:07:54.251 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:07:54.252767 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:08:04.507 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:08:04.508494 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:08:04.522 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:08:04.523245 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:08:14.752 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:08:14.752137 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:08:14.769 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:08:14.769234 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:08:25.002 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:08:25.003781 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:08:25.015 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:08:25.016826 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:08:35.268 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:08:35.269048 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:08:35.280 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:08:35.281206 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:08:45.508 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:08:45.508976 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:08:45.524 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:08:45.525623 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:08:55.764 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:08:55.765542 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:08:55.780 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:08:55.781940 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:09:06.013 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:09:06.014458 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:09:06.027 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:09:06.029039 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:09:16.289 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:09:16.290203 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:09:16.303 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:09:16.304467 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:09:26.522 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:09:26.523698 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:09:26.539 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:09:26.540561 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:09:36.764 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:09:36.765463 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:09:36.778 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:09:36.779811 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:09:47.000 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:09:47.001362 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:09:47.014 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:09:47.015991 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:09:57.306 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:09:57.307820 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:09:57.323 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:09:57.324270 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:10:07.561 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:10:07.562886 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:10:07.577 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:10:07.578855 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:10:17.800 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:10:17.801873 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:10:17.817 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:10:17.818709 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:10:28.041 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:10:28.042644 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:10:28.057 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:10:28.059161 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:10:38.279 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:10:38.280820 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:10:38.311 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:10:38.312448 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:10:48.527 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:10:48.528052 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:10:48.541 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:10:48.542938 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:10:58.787 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:10:58.787941 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:10:58.825 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:10:58.826704 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:11:09.079 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:11:09.079657 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:11:09.092 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:11:09.093497 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:11:19.317 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:11:19.318313 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:11:19.330 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:11:19.331893 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:11:29.553 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:11:29.554267 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:11:29.567 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:11:29.568063 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:11:39.789 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:11:39.790498 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:11:39.815 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:11:39.816533 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:11:50.037 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:11:50.038437 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:11:50.052 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:11:50.053300 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:12:00.274 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:12:00.274928 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:12:00.307 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:12:00.308306 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:12:10.576 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:12:10.576930 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:12:10.586 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:12:10.587839 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:12:20.813 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:12:20.813390 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:12:20.826 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:12:20.827346 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:12:31.073 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:12:31.073628 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:12:31.085 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:12:31.086170 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:12:41.313 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:12:41.313382 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:12:41.328 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:12:41.325897 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:12:51.548 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:12:51.548359 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:12:51.561 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:12:51.562045 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:13:01.815 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:13:01.815393 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:13:01.854 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:13:01.854780 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:13:12.076 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:13:12.077012 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:13:12.090 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:13:12.091032 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:13:22.315 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:13:22.315930 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:13:22.331 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:13:22.331831 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:13:32.547 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:13:32.547267 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:13:32.563 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:13:32.564254 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:13:42.786 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:13:42.786883 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:13:42.795 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:13:42.796586 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:13:53.029 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:13:53.029886 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:13:53.045 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:13:53.045703 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:14:03.264 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:14:03.265033 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:14:03.280 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:14:03.280335 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:14:03.280 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:14:03.280369 I | op-k8sutil: Reporting Event rook-ceph:rook-ceph Warning:ReconcileFailed:CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:14:13.500 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:14:13.501121 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:14:13.514 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:14:13.515022 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:14:23.733 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:14:23.733647 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:14:23.744 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:14:23.745093 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:14:33.973 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:14:33.973947 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:14:33.988 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:14:33.989026 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:14:44.231 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:14:44.231815 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:14:44.243 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:14:44.244420 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:14:54.475 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:14:54.475617 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:14:54.491 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:14:54.492051 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:15:04.713 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:15:04.713277 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:15:04.727 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:15:04.727905 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:15:14.963 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:15:14.964143 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:15:14.976 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:15:14.976348 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:15:25.195 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:15:25.195470 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:15:25.209 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:15:25.210115 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:15:35.442 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:15:35.442442 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:15:35.453 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:15:35.453253 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:15:45.699 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:15:45.699373 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:15:45.711 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:15:45.711731 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:15:55.934 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:15:55.934240 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:15:55.948 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:15:55.948860 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:16:06.174 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:16:06.174484 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:16:06.189 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:16:06.190235 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:16:16.416 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:16:16.415862 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:16:16.429 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:16:16.430120 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:16:26.587 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:16:26.587632 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 22:16:26.534605548 +0000 UTC m=+13270.804148071 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T22:16:26.654 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:16:26.654887 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:16:26.666 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:16:26.666741 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:16:36.891 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:16:36.891139 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:16:36.904 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:16:36.904919 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:16:47.131 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:16:47.131289 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:16:47.162 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:16:47.162532 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:16:57.455 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:16:57.455222 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:16:57.466 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:16:57.466615 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:17:07.722 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:17:07.722118 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:17:07.736 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:17:07.736617 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:17:17.955 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:17:17.955681 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:17:17.972 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:17:17.972789 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:17:28.194 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:17:28.194013 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:17:28.208 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:17:28.209102 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:17:38.476 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:17:38.476265 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:17:38.490 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:17:38.490201 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:17:48.714 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:17:48.714273 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:17:48.727 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:17:48.727580 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:17:58.951 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:17:58.951638 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:17:58.971 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:17:58.970621 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:18:09.198 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:18:09.197914 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:18:09.214 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:18:09.214533 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:18:19.437 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:18:19.437486 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:18:19.457 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:18:19.454977 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:18:29.692 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:18:29.691913 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:18:29.705 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:18:29.705331 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:18:39.926 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:18:39.925726 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:18:39.940 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:18:39.940856 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:18:50.166 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:18:50.165900 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:18:50.181 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:18:50.181468 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:19:00.403 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:19:00.403458 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:19:00.419 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:19:00.419821 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:19:10.635 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:19:10.635353 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:19:10.649 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:19:10.649143 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:19:20.868 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:19:20.868241 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:19:20.882 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:19:20.882096 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:19:31.106 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:19:31.105518 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:19:31.120 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:19:31.120242 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:19:41.374 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:19:41.374063 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:19:41.390 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:19:41.390068 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:19:51.614 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:19:51.614041 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:19:51.637 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:19:51.637033 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:20:01.857 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:20:01.856622 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:20:01.872 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:20:01.872683 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:20:12.095 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:20:12.095310 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:20:12.110 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:20:12.110887 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:20:22.332 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:20:22.332121 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:20:22.346 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:20:22.346735 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:20:32.573 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:20:32.572720 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:20:32.586 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:20:32.586791 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:20:42.813 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:20:42.812585 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:20:42.827 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:20:42.827897 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:20:53.061 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:20:53.060692 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:20:53.078 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:20:53.077523 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:21:03.302 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:03.301763 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:21:03.316 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:03.316502 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:21:13.536 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:13.536341 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:21:13.548 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:13.548771 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:21:23.774 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:23.773586 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:21:23.795 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:23.788207 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:21:34.015 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:34.015052 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:21:34.025 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:34.025000 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:21:42.522 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:42.522322 I | op-mon: parsing mon endpoints: b=10.110.64.66:6789,c=10.109.10.123:6789,a=10.101.136.169:6789 2022-03-08T22:21:43.192 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:43.190752 E | ceph-object-controller: failed to reconcile CephObjectStore "rook-ceph/foo". failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T22:21:43.181+0000 7f4f1a865340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T22:21:43.192 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T22:21:43.181+0000 7f4f1a865340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T22:21:43.192 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T22:21:43.181+0000 7f4f1a865340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T22:21:43.193 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T22:21:43.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:43.190783 I | op-k8sutil: Reporting Event rook-ceph:foo Warning:ReconcileFailed:failed to check for object buckets. failed to get admin ops API context: failed to create or retrieve rgw admin ops user: failed to create object user "rgw-admin-ops-user". error code 1 for object store "foo": failed to create s3 user. 2022-03-08T22:21:43.181+0000 7f4f1a865340 0 failed reading zonegroup info: ret -2 (2) No such file or directory 2022-03-08T22:21:43.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T22:21:43.181+0000 7f4f1a865340 0 ERROR: failed to start notify service ((2) No such file or directory 2022-03-08T22:21:43.193 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08T22:21:43.181+0000 7f4f1a865340 0 ERROR: failed to init services (ret=(2) No such file or directory) 2022-03-08T22:21:43.194 INFO:tasks.rook.operator.smithi002.stdout:couldn't init storage provider. : exit status 5 2022-03-08T22:21:44.246 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:44.245997 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:21:44.268 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:44.268508 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:21:54.495 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:54.495211 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:21:54.515 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:21:54.515639 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:22:04.738 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:22:04.738209 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:22:04.753 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:22:04.753178 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:22:14.970 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:22:14.969968 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:22:15.035 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:22:15.035028 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:22:25.272 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:22:25.271793 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:22:25.299 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:22:25.299710 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:22:35.568 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:22:35.568495 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:22:35.579 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:22:35.579521 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:22:45.800 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:22:45.799995 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:22:45.815 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:22:45.815571 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:22:56.042 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:22:56.041814 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:22:56.058 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:22:56.057537 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:23:06.335 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:23:06.334360 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:23:06.345 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:23:06.345117 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:23:16.568 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:23:16.567519 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:23:16.583 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:23:16.582810 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:23:26.814 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:23:26.814152 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:23:26.831 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:23:26.831351 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:23:37.107 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:23:37.106732 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:23:37.126 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:23:37.125806 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:23:47.346 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:23:47.346287 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:23:47.363 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:23:47.363148 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:23:57.582 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:23:57.581939 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:23:57.593 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:23:57.593210 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:24:07.818 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:24:07.818231 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:24:07.836 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:24:07.835958 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:24:18.063 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:24:18.062740 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:24:18.080 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:24:18.079731 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:24:28.308 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:24:28.307581 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:24:28.321 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:24:28.321654 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:24:38.553 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:24:38.552651 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:24:38.562 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:24:38.562403 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:24:48.784 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:24:48.784354 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:24:48.799 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:24:48.799266 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:24:59.020 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:24:59.019172 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:24:59.038 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:24:59.036044 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:25:09.262 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:25:09.260093 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:25:09.274 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:25:09.273793 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:25:19.499 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:25:19.498889 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:25:19.514 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:25:19.514592 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:25:29.737 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:25:29.736517 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:25:29.752 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:25:29.752408 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:25:40.009 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:25:40.008721 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:25:40.021 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:25:40.021446 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:25:50.246 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:25:50.246255 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:25:50.263 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:25:50.263269 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:26:00.490 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:26:00.490140 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:26:00.505 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:26:00.505163 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:26:10.729 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:26:10.729060 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:26:10.744 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:26:10.744626 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:26:20.977 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:26:20.976953 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:26:20.991 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:26:20.990857 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:26:31.213 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:26:31.213312 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:26:31.246 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:26:31.246522 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:26:41.508 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:26:41.507292 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:26:41.526 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:26:41.524246 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:26:51.796 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:26:51.795709 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:26:51.809 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:26:51.809041 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:27:02.031 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:27:02.030486 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:27:02.107 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:27:02.102757 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:27:12.325 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:27:12.325084 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:27:12.341 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:27:12.340708 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:27:22.565 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:27:22.565222 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:27:22.579 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:27:22.579870 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:27:32.805 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:27:32.804757 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:27:32.819 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:27:32.818705 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:27:43.039 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:27:43.039304 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:27:43.057 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:27:43.056898 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:27:53.279 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:27:53.279082 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:27:53.296 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:27:53.295902 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:28:03.520 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:28:03.519623 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:28:03.534 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:28:03.533926 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:28:13.785 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:28:13.784423 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:28:13.811 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:28:13.799290 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:28:24.022 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:28:24.021816 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:28:24.031 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:28:24.031791 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:28:34.250 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:28:34.250374 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:28:34.265 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:28:34.265190 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:28:44.489 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:28:44.488791 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:28:44.505 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:28:44.505646 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:28:54.774 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:28:54.774295 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:28:54.794 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:28:54.793991 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:29:05.018 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:29:05.018040 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:29:05.034 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:29:05.034075 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:29:15.262 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:29:15.261583 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:29:15.281 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:29:15.281726 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:29:25.505 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:29:25.504567 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:29:25.517 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:29:25.517040 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:29:35.737 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:29:35.737063 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:29:35.751 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:29:35.750788 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:29:45.972 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:29:45.972180 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:29:45.987 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:29:45.987050 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:29:56.213 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:29:56.212823 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:29:56.251 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:29:56.245398 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:30:06.470 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:30:06.469314 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:30:06.483 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:30:06.482853 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:30:16.710 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:30:16.709867 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:30:16.728 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:30:16.727603 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:30:26.975 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:30:26.974712 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:30:26.989 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:30:26.989367 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:30:37.272 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:30:37.272128 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:30:37.286 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:30:37.285755 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:30:47.568 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:30:47.567669 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:30:47.579 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:30:47.578607 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:30:57.836 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:30:57.835628 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:30:57.852 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:30:57.851900 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:31:08.071 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:31:08.071065 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:31:08.085 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:31:08.085453 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:31:18.308 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:31:18.307732 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:31:18.323 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:31:18.323529 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:31:28.547 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:31:28.546569 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:31:28.561 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:31:28.561132 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:31:38.786 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:31:38.786352 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:31:38.802 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:31:38.801751 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:31:49.027 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:31:49.027062 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:31:49.051 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:31:49.050823 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:31:59.325 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:31:59.324700 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:31:59.342 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:31:59.342208 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:32:09.564 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:32:09.563579 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:32:09.580 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:32:09.580044 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:32:19.805 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:32:19.805119 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:32:19.819 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:32:19.819725 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:32:30.041 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:32:30.040925 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:32:30.055 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:32:30.055649 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:32:40.282 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:32:40.282058 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:32:40.298 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:32:40.298481 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:32:40.632 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:32:40.631563 E | ceph-spec: failed to update cluster condition to {Type:Ready Status:True Reason:ClusterCreated Message:Cluster created successfully LastHeartbeatTime:2022-03-08 22:32:40.618647761 +0000 UTC m=+14244.888190241 LastTransitionTime:2022-03-08 18:39:08 +0000 UTC}. failed to update object "rook-ceph/rook-ceph" status: Operation cannot be fulfilled on cephclusters.ceph.rook.io "rook-ceph": the object has been modified; please apply your changes to the latest version and try again 2022-03-08T22:32:50.522 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:32:50.521613 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:32:50.535 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:32:50.535344 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:33:00.759 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:33:00.758566 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:33:00.775 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:33:00.775661 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:33:10.994 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:33:10.994052 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:33:11.009 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:33:11.009555 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:33:21.230 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:33:21.229756 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:33:21.245 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:33:21.245332 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:33:31.474 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:33:31.473609 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:33:31.487 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:33:31.487510 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:33:41.776 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:33:41.775675 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:33:41.796 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:33:41.791768 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:33:52.016 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:33:52.016014 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:33:52.030 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:33:52.030463 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:34:02.253 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:34:02.252402 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:34:02.267 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:34:02.266769 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:34:12.493 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:34:12.491943 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:34:12.507 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:34:12.507141 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:34:22.730 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:34:22.729431 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:34:22.745 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:34:22.745409 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:34:32.986 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:34:32.985405 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:34:32.997 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:34:32.997156 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:34:43.241 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:34:43.240509 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:34:43.251 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:34:43.251196 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:34:53.532 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:34:53.530844 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:34:53.547 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:34:53.546889 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:35:03.769 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:35:03.768398 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:35:03.782 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:35:03.782383 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:35:14.005 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:35:14.005232 I | ceph-cluster-controller: CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-08T22:35:14.020 INFO:tasks.rook.operator.smithi002.stdout:2022-03-08 22:35:14.020520 E | ceph-cluster-controller: failed to reconcile CephCluster "rook-ceph/rook-ceph". CephCluster "rook-ceph/rook-ceph" will not be deleted until all dependents are removed: CephObjectStores: [foo] 2022-03-09T00:55:38.585 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2022-03-09T00:55:38.620 DEBUG:teuthology.task.console_log:Killing console logger for smithi002 2022-03-09T00:55:38.621 DEBUG:teuthology.task.console_log:Killing console logger for smithi066 2022-03-09T00:55:38.622 DEBUG:teuthology.task.console_log:Killing console logger for smithi137 2022-03-09T00:55:38.622 DEBUG:teuthology.exit:Finished running handlers