2020-02-14T15:17:01.569 INFO:root:teuthology version: 1.0.0-ee19bca 2020-02-14T15:17:01.570 DEBUG:teuthology.run:Teuthology command: teuthology /tmp/teuthology-worker.9f1F2E.tmp -- --verbose --lock --description rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack.yaml start.yaml} 1-install/nautilus.yaml 2-partial-upgrade/firsthalf.yaml 3-thrash/default.yaml 4-workload/{rbd-cls.yaml rbd-import-export.yaml readwrite.yaml snaps-few-objects.yaml} 5-workload/{radosbench.yaml rbd_api.yaml} 6-finish-upgrade.yaml 7-octopus.yaml 8-workload/{rbd-python.yaml snaps-many-objects.yaml} bluestore-bitmap.yaml thrashosds-health.yaml ubuntu_latest.yaml} --name kchai-2020-02-14_15:08:49-rados-wip-kefu-testing-2020-02-14-2045-distro-basic-smithi --block --owner scheduled_kchai@teuthology --archive /home/teuthworker/archive/kchai-2020-02-14_15:08:49-rados-wip-kefu-testing-2020-02-14-2045-distro-basic-smithi/4763790 2020-02-14T15:17:01.690 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/ 2020-02-14T15:17:01.801 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/kchai-2020-02-14_15:08:49-rados-wip-kefu-testing-2020-02-14-2045-distro-basic-smithi/4763790 branch: wip-kefu-testing-2020-02-14-2045 description: rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack.yaml start.yaml} 1-install/nautilus.yaml 2-partial-upgrade/firsthalf.yaml 3-thrash/default.yaml 4-workload/{rbd-cls.yaml rbd-import-export.yaml readwrite.yaml snaps-few-objects.yaml} 5-workload/{radosbench.yaml rbd_api.yaml} 6-finish-upgrade.yaml 7-octopus.yaml 8-workload/{rbd-python.yaml snaps-many-objects.yaml} bluestore-bitmap.yaml thrashosds-health.yaml ubuntu_latest.yaml} email: null first_in_suite: false job_id: '4763790' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'Run ceph on two nodes, with a separate client-only node. Use xfs beneath the osds. ' - desc: install ceph/nautilus latest - desc: 'install upgrade ceph/-x on one node only 1st half restart : osd.0,1,2,3,4,5 ' - desc: 'randomly kill and revive osd small chance to increase the number of pgs ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'librbd C and C++ api tests ' - desc: 'install upgrade on remaining node restartin remaining osds ' - desc: 'librbd python api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' name: kchai-2020-02-14_15:08:49-rados-wip-kefu-testing-2020-02-14-2045-distro-basic-smithi nuke-on-error: true openstack: - machine: disk: 100 - volumes: count: 3 size: 30 os_type: ubuntu os_version: '18.04' overrides: admin_socket: branch: wip-kefu-testing-2020-02-14-2045 ceph: conf: global: bluestore warn on no per pool omap: false mon pg warn min per osd: 0 ms bind msgr2: false ms dump corrupt message level: 0 mds: debug mds: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 20 debug bluestore: 20 debug filestore: 20 debug journal: 20 debug ms: 1 debug osd: 25 debug rocksdb: 10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs log-whitelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(MON_DOWN\) - \(MGR_DOWN\) - slow request - but it is still running - objects unfound and apparently lost - log bound mismatch - overall HEALTH_ - \(FS_DEGRADED\) - \(MDS_ - overall HEALTH_ - \(OSDMAP_FLAGS\) - \(OSD_ - \(PG_ - \(POOL_ - \(CACHE_POOL_ - \(SMALLER_PGP_NUM\) - \(OBJECT_ - \(SLOW_OPS\) - \(REQUEST_SLOW\) - \(TOO_FEW_PGS\) - slow requests mon_bind_addrvec: false mon_bind_msgr2: false sha1: a075f279f1948ef2c8122c5ac06d522688c3eb90 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 20 debug bluestore: 20 debug rocksdb: 10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: sha1: a075f279f1948ef2c8122c5ac06d522688c3eb90 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-kefu-testing-2020-02-14-2045 sha1: a075f279f1948ef2c8122c5ac06d522688c3eb90 owner: scheduled_kchai@teuthology priority: 30 repo: git://git.ceph.com/ceph-ci.git roles: - - mon.a - mgr.x - mds.a - osd.0 - osd.1 - osd.2 - osd.3 - - mon.b - mgr.y - osd.4 - osd.5 - osd.6 - osd.7 - - mon.c - osd.8 - osd.9 - osd.10 - osd.11 - - client.0 sha1: a075f279f1948ef2c8122c5ac06d522688c3eb90 split_tasks: sequential: - thrashosds: aggressive_pg_num_changes: false chance_pgnum_grow: 1 chance_pgpnum_fix: 1 disable_objectstore_tool_tests: true timeout: 1200 - print: '**** done thrashosds 3-thrash' - workunit: branch: nautilus clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: nautilus clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' - full_sequential: - radosbench: clients: - client.0 time: 150 - radosbench: clients: - client.0 time: 150 - radosbench: clients: - client.0 time: 150 - radosbench: clients: - client.0 time: 150 - radosbench: clients: - client.0 time: 150 - radosbench: clients: - client.0 time: 150 - radosbench: clients: - client.0 time: 150 - radosbench: clients: - client.0 time: 150 - radosbench: clients: - client.0 time: 150 - radosbench: clients: - client.0 time: 150 - radosbench: clients: - client.0 time: 150 - print: '**** done radosbench 7-workload' - workunit: branch: nautilus clients: client.0: - rbd/test_librbd.sh - print: '**** done rbd/test_librbd.sh 7-workload' suite: rados suite_branch: wip-kefu-testing-2020-02-14-2045 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_wip-kefu-testing-2020-02-14-2045/qa suite_relpath: qa suite_repo: git://git.ceph.com/ceph-ci.git suite_sha1: a075f279f1948ef2c8122c5ac06d522688c3eb90 tasks: - install: branch: nautilus exclude_packages: - ceph-mgr-cephadm - cephadm - print: '**** done install nautilus' - ceph: null - print: '**** done ceph' - exec: mon.a: - sudo ceph osd pool create test-hit-set-base 32 - sudo ceph osd pool create test-hit-set-cache 32 - sudo ceph osd tier add test-hit-set-base test-hit-set-cache - sudo ceph osd pool set test-hit-set-cache hit_set_type bloom - sudo ceph osd pool set test-hit-set-cache hit_set_count 32 - sudo ceph osd pool set test-hit-set-cache hit_set_period 15 - rados -p test-hit-set-cache bench 30 write -b 1 - install.upgrade: mon.a: null mon.b: null - print: '**** done install.upgrade osd.0' - ceph.restart: daemons: - mgr.x - mgr.y - ceph.restart: daemons: - mon.a - mon.b mon-health-to-clog: false wait-for-healthy: false - ceph.restart: daemons: - osd.0 - osd.1 - osd.2 - osd.3 - osd.4 - osd.5 - osd.6 - osd.7 wait-for-healthy: false - print: '**** done ceph.restart 1st 2/3s' - parallel: - split_tasks - install.upgrade: mon.c: null - ceph.restart: daemons: - mon.c - mgr.x - mgr.y wait-for-healthy: false wait-for-up: true - ceph.restart: daemons: - osd.8 - osd.9 - osd.10 - osd.11 wait-for-healthy: false wait-for-up: true - ceph.restart: daemons: - mds.a wait-for-healthy: false wait-for-up: true - exec: mon.a: - ceph mon enable-msgr2 - install.upgrade: client.0: null - exec: mon.a: - sudo ceph osd pool set test-hit-set-cache hit_set_count 4 - rados -p test-hit-set-cache bench 5 write -b 1 - exec: osd.0: - ceph osd require-osd-release octopus - ceph osd set-require-min-compat-client octopus - for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done - ceph.healthy: null - workunit: clients: client.0: - rbd/test_librbd_python.sh - print: '**** done rbd/test_librbd_python.sh 9-workload' - rados: clients: - client.0 objects: 500 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false teuthology_branch: master tube: smithi verbose: false worker_log: /home/teuthworker/archive/worker_logs/worker.smithi.2502 2020-02-14T15:17:01.802 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_wip-kefu-testing-2020-02-14-2045/qa; will attempt to use it 2020-02-14T15:17:01.802 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_wip-kefu-testing-2020-02-14-2045/qa/tasks 2020-02-14T15:17:01.803 INFO:teuthology.run_tasks:Running task internal.check_packages... 2020-02-14T15:17:01.804 INFO:teuthology.task.internal:Checking packages... 2020-02-14T15:17:01.822 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'basic' and ceph hash 'a075f279f1948ef2c8122c5ac06d522688c3eb90' 2020-02-14T15:17:01.822 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2020-02-14T15:17:01.822 INFO:teuthology.packaging:ref: None 2020-02-14T15:17:01.822 INFO:teuthology.packaging:tag: None 2020-02-14T15:17:01.823 INFO:teuthology.packaging:branch: wip-kefu-testing-2020-02-14-2045 2020-02-14T15:17:01.823 INFO:teuthology.packaging:sha1: a075f279f1948ef2c8122c5ac06d522688c3eb90 2020-02-14T15:17:01.823 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F18.04%2Fx86_64&ref=wip-kefu-testing-2020-02-14-2045 2020-02-14T15:17:01.979 INFO:teuthology.task.internal:Found packages for ceph version 15.1.0-735-ga075f27-1bionic 2020-02-14T15:17:01.979 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2020-02-14T15:17:02.001 INFO:teuthology.task.internal:no buildpackages task found 2020-02-14T15:17:02.001 INFO:teuthology.run_tasks:Running task internal.lock_machines... 2020-02-14T15:17:02.017 INFO:teuthology.task.internal.lock_machines:Locking machines... 2020-02-14T15:17:02.017 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/ 2020-02-14T15:17:02.083 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:17:12.104 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:17:22.125 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:17:32.144 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:17:42.164 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:17:52.184 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:18:02.202 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:18:12.225 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:18:22.244 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:18:32.266 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:18:42.283 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:18:52.303 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:19:02.320 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:19:12.339 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:19:22.359 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:19:32.378 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:19:42.395 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:19:52.416 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:20:02.425 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:20:12.445 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:20:22.463 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:20:32.483 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:20:42.511 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:20:52.524 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:21:02.545 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:21:12.564 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:21:22.583 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:21:32.600 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:21:42.615 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:21:52.635 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:22:02.654 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:22:12.677 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:22:22.694 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:22:32.711 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:22:42.732 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:22:52.746 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:23:02.764 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:23:12.783 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:23:22.800 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:23:32.821 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:23:42.836 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:23:52.855 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:24:02.872 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:24:12.894 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:24:22.911 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:24:32.929 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:24:42.946 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:24:52.960 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:25:02.976 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:25:12.999 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:25:23.014 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:25:33.035 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:25:43.044 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 1)... 2020-02-14T15:25:53.056 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 1)... 2020-02-14T15:26:03.077 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 1)... 2020-02-14T15:26:13.095 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 1)... 2020-02-14T15:26:23.110 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 1)... 2020-02-14T15:26:33.127 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 1)... 2020-02-14T15:26:43.144 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:26:53.164 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:27:03.186 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:27:13.211 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:27:23.232 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:27:33.254 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:27:43.276 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:27:53.292 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:28:03.314 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:28:13.334 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:28:23.352 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:28:33.373 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:28:43.393 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:28:53.415 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:29:03.438 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:29:13.471 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:29:23.492 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:29:33.507 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:29:43.526 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:29:53.545 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:30:03.564 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:30:13.586 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:30:23.604 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:30:33.625 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:30:43.652 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:30:53.672 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:31:03.687 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:31:13.698 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:31:23.714 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:31:33.732 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:31:43.744 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:31:53.760 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:32:03.780 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:32:13.792 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:32:23.810 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:32:33.847 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:32:43.865 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:32:53.885 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:33:03.903 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:33:13.921 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:33:23.940 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:33:33.970 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 4)... 2020-02-14T15:33:43.988 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:33:54.003 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:34:04.023 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:34:14.043 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:34:24.065 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:34:34.085 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:34:44.105 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 0)... 2020-02-14T15:34:54.122 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:35:04.140 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:35:14.156 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:35:24.173 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:35:34.193 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:35:44.216 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:35:54.238 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:36:04.262 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:36:14.283 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:36:24.302 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:36:34.320 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:36:44.354 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:36:54.372 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 2)... 2020-02-14T15:37:04.399 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 5)... 2020-02-14T15:37:14.418 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:37:24.440 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:37:34.459 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:37:44.472 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:37:54.485 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:38:04.502 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:38:14.521 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:38:24.540 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:38:34.554 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:38:44.571 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:38:54.586 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 4, have 3)... 2020-02-14T15:39:04.607 DEBUG:teuthology.lock.ops:lock_many request: {'count': 4, 'locked_by': 'scheduled_kchai@teuthology', 'description': '/home/teuthworker/archive/kchai-2020-02-14_15:08:49-rados-wip-kefu-testing-2020-02-14-2045-distro-basic-smithi/4763790', 'machine_type': 'smithi'} 2020-02-14T15:39:04.633 DEBUG:teuthology.lock.ops:locked ubuntu@smithi064.front.sepia.ceph.com, ubuntu@smithi141.front.sepia.ceph.com, ubuntu@smithi083.front.sepia.ceph.com, ubuntu@smithi140.front.sepia.ceph.com 2020-02-14T15:39:04.769 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=, signals=[15]) 2020-02-14T15:39:05.025 INFO:teuthology.provision.fog.smithi140:Scheduling deploy of ubuntu 18.04 2020-02-14T15:39:05.053 INFO:teuthology.provision.fog.smithi064:Scheduling deploy of ubuntu 18.04 2020-02-14T15:39:05.057 INFO:teuthology.provision.fog.smithi083:Scheduling deploy of ubuntu 18.04 2020-02-14T15:39:05.068 INFO:teuthology.provision.fog.smithi141:Scheduling deploy of ubuntu 18.04 2020-02-14T15:39:05.383 INFO:teuthology.orchestra.console:Power off smithi140 2020-02-14T15:39:05.384 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi140.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2020-02-14T15:39:05.407 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi140.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:05.430 INFO:teuthology.orchestra.console:Power off smithi083 2020-02-14T15:39:05.431 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2020-02-14T15:39:05.458 INFO:teuthology.orchestra.console:Power off smithi064 2020-02-14T15:39:05.459 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2020-02-14T15:39:05.469 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:05.484 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:05.504 INFO:teuthology.orchestra.console:Power off smithi141 2020-02-14T15:39:05.504 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi141.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2020-02-14T15:39:05.525 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi141.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:05.615 INFO:teuthology.orchestra.console:Power off for smithi064 completed 2020-02-14T15:39:05.715 INFO:teuthology.orchestra.console:Power on smithi064 2020-02-14T15:39:05.716 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power on 2020-02-14T15:39:05.736 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:09.473 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi140.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:09.515 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:09.549 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi141.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:09.762 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:13.586 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi140.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:13.626 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:13.658 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi141.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:13.871 INFO:teuthology.orchestra.console:Power off for smithi141 completed 2020-02-14T15:39:13.872 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:13.972 INFO:teuthology.orchestra.console:Power on smithi141 2020-02-14T15:39:13.973 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi141.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power on 2020-02-14T15:39:13.999 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi141.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:14.098 INFO:teuthology.orchestra.console:Power on for smithi064 completed 2020-02-14T15:39:14.201 INFO:teuthology.provision.fog.smithi064:Waiting for deploy to finish 2020-02-14T15:39:17.698 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi140.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:17.749 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:17.912 INFO:teuthology.orchestra.console:Power off for smithi140 completed 2020-02-14T15:39:17.959 INFO:teuthology.orchestra.console:Power off for smithi083 completed 2020-02-14T15:39:18.014 INFO:teuthology.orchestra.console:Power on smithi140 2020-02-14T15:39:18.014 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi140.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power on 2020-02-14T15:39:18.027 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi141.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:18.042 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi140.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:18.061 INFO:teuthology.orchestra.console:Power on smithi083 2020-02-14T15:39:18.061 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power on 2020-02-14T15:39:18.097 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:22.089 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi140.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:22.124 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:22.143 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi141.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:22.299 INFO:teuthology.orchestra.console:Power on for smithi140 completed 2020-02-14T15:39:22.355 INFO:teuthology.orchestra.console:Power on for smithi141 completed 2020-02-14T15:39:22.401 INFO:teuthology.provision.fog.smithi140:Waiting for deploy to finish 2020-02-14T15:39:22.457 INFO:teuthology.provision.fog.smithi141:Waiting for deploy to finish 2020-02-14T15:39:26.234 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2020-02-14T15:39:26.444 INFO:teuthology.orchestra.console:Power on for smithi083 completed 2020-02-14T15:39:26.545 INFO:teuthology.provision.fog.smithi083:Waiting for deploy to finish 2020-02-14T15:42:23.804 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi141.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:42:30.655 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi064.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:42:32.855 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi141.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:42:38.893 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi140.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:42:39.740 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi064.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:42:41.912 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi141.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:42:42.948 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi083.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:42:47.961 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi140.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:42:48.791 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi064.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:42:50.968 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi141.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:42:52.030 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi083.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:42:57.020 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi140.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:42:57.849 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi064.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:00.023 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi141.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:01.110 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi083.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:06.103 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi140.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:06.907 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi064.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:09.079 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi141.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:10.167 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi083.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:15.164 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi140.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:15.959 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi064.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:18.136 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi141.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:19.222 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi083.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:24.215 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi140.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:25.015 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi064.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:28.279 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi083.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:33.271 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi140.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:34.075 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi064.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:37.334 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi083.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:42.328 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi140.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:43.163 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi064.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:46.395 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi083.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:49.738 ERROR:paramiko.transport:Socket exception: Connection reset by peer (104) 2020-02-14T15:43:52.214 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi064.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:55.447 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi083.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:55.739 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi141.front.sepia.ceph.com', 'timeout': 60} 2020-02-14T15:43:55.751 ERROR:paramiko.transport:Socket exception: Connection reset by peer (104) 2020-02-14T15:43:59.609 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2020-02-14T15:43:59.610 DEBUG:teuthology.task.console_log:Killing console logger for smithi064 2020-02-14T15:43:59.610 DEBUG:teuthology.task.console_log:Killing console logger for smithi140 2020-02-14T15:43:59.610 DEBUG:teuthology.task.console_log:Killing console logger for smithi141 2020-02-14T15:43:59.611 DEBUG:teuthology.task.console_log:Killing console logger for smithi083 2020-02-14T15:43:59.611 DEBUG:teuthology.exit:Finished running handlers