2020-01-27T18:37:37.241 INFO:root:teuthology version: 1.0.0-7063d1e 2020-01-27T18:37:37.241 DEBUG:teuthology.run:Teuthology command: teuthology /tmp/teuthology-worker.qGsIMq.tmp -- --verbose --lock --description upgrade:mimic-x/stress-split-erasure-code/{0-cluster/{openstack.yaml start.yaml} 1-luminous-install/mimic.yaml 1.1-pg-log-overrides/normal_pg_log.yaml 2-partial-upgrade/firsthalf.yaml 3-thrash/default.yaml 4-ec-workload.yaml 5-finish-upgrade.yaml 7-final-workload.yaml objectstore/filestore-xfs.yaml supported-all-distro/rhel_7.yaml thrashosds-health.yaml} --name dzafman-2020-01-25_01:21:34-upgrade:mimic-x-master-distro-basic-smithi --block --owner scheduled_dzafman@ubuntu-1804 --archive /home/teuthworker/archive/dzafman-2020-01-25_01:21:34-upgrade:mimic-x-master-distro-basic-smithi/4711859 2020-01-27T18:37:37.296 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/ 2020-01-27T18:37:37.364 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/dzafman-2020-01-25_01:21:34-upgrade:mimic-x-master-distro-basic-smithi/4711859 branch: master description: upgrade:mimic-x/stress-split-erasure-code/{0-cluster/{openstack.yaml start.yaml} 1-luminous-install/mimic.yaml 1.1-pg-log-overrides/normal_pg_log.yaml 2-partial-upgrade/firsthalf.yaml 3-thrash/default.yaml 4-ec-workload.yaml 5-finish-upgrade.yaml 7-final-workload.yaml objectstore/filestore-xfs.yaml supported-all-distro/rhel_7.yaml thrashosds-health.yaml} email: dzafman@redhat.com first_in_suite: false job_id: '4711859' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'Run ceph on two nodes, with a separate client-only node. Use xfs beneath the osds. #Note-To enable RHEL runs on ovh nodes, add the following to overrides #ansible.cephlab: # skip_tags: entitlements,packages,repos ' - desc: install ceph/mimic latest - desc: 'install upgrade ceph/-x on 2/3 of cluster restart : mons, osd 0-7 ' - desc: 'randomly kill and revive osd small chance to increase the number of pgs ' - desc: 'randomized correctness test for rados operations on an erasure coded pool ' - desc: 'randomized correctness test for rados operations on an erasure coded pool using the jerasure plugin with k=3 and m=1 ' name: dzafman-2020-01-25_01:21:34-upgrade:mimic-x-master-distro-basic-smithi nuke-on-error: true openstack: - machine: disk: 100 - volumes: count: 4 size: 30 os_type: rhel os_version: '7.7' overrides: admin_socket: branch: master ceph: conf: global: enable experimental unrecoverable data corrupting features: '*' mon warn on msgr2 not enabled: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon warn on osd down out interval zero: false osd: debug filestore: 20 debug journal: 20 debug ms: 1 debug osd: 25 osd objectstore: filestore osd sloppy crc: true fs: xfs log-whitelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - overall HEALTH_ - \(MON_DOWN\) - \(MGR_DOWN\) - \(MON_MSGR2_NOT_ENABLED\) - but it is still running - wrongly marked me down - objects unfound and apparently lost - log bound mismatch - overall HEALTH_ - \(OSDMAP_FLAGS\) - \(OSD_ - \(PG_ - \(POOL_ - \(CACHE_POOL_ - \(SMALLER_PGP_NUM\) - \(OBJECT_ - \(SLOW_OPS\) - \(REQUEST_SLOW\) - \(TOO_FEW_PGS\) - slow requests mon_bind_addrvec: false mon_bind_msgr2: false sha1: 9c19d0edb97c85432761f769828070f563587117 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 osd: osd objectstore: filestore osd sloppy crc: true filestore: true fs: xfs install: ceph: sha1: 9c19d0edb97c85432761f769828070f563587117 workunit: branch: master sha1: 35b0b52153206fd6178afe584a2cd9c0ef2bcc05 owner: scheduled_dzafman@ubuntu-1804 priority: 50 repo: git://git.ceph.com/ceph-ci.git roles: - - mon.a - mgr.x - osd.0 - osd.1 - osd.2 - osd.3 - - mon.b - osd.4 - osd.5 - osd.6 - osd.7 - - mon.c - - osd.8 - osd.9 - osd.10 - osd.11 - - client.0 sha1: 9c19d0edb97c85432761f769828070f563587117 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 min_in: 4 timeout: 1200 - print: '**** done thrashosds 3-thrash' - rados: clients: - client.0 ec_pool: true objects: 50 op_weights: append: 100 copy_from: 50 delete: 50 read: 100 rmattr: 25 rollback: 50 setattr: 25 snap_create: 50 snap_remove: 50 write: 0 ops: 4000 write_append_excl: false - print: '**** done rados ec task' suite: upgrade:mimic-x suite_branch: master suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_master/qa suite_relpath: qa suite_repo: git://git.ceph.com/ceph-ci.git suite_sha1: 35b0b52153206fd6178afe584a2cd9c0ef2bcc05 tasks: - install: branch: mimic exclude_packages: - librados3 - ceph-mgr-dashboard - ceph-mgr-diskprediction-local - ceph-mgr-diskprediction-cloud - ceph-mgr-rook - ceph-mgr-ssh - ceph-daemon extra_packages: - librados2 - print: '**** done install mimic' - ceph: conf: global: bluestore warn on no per pool omap: false bluestore_warn_on_legacy_statfs: false - exec: osd.0: - ceph osd require-osd-release mimic - ceph osd set-require-min-compat-client mimic - print: '**** done ceph' - rgw: - client.0 - print: '**** done => started rgw client.0' - install.upgrade: mon.a: null mon.b: null mon.c: null - print: '**** done install.upgrade first 3 nodes' - ceph.restart: daemons: - mon.a - mon.b - mon.c - mgr.x - osd.0 - osd.1 - osd.2 - osd.3 - osd.4 - osd.5 - osd.6 - osd.7 mon-health-to-clog: false - print: '**** done ceph.restart all mons and 2/3 of osds' - parallel: - stress-tasks - install.upgrade: client.0: null osd.8: null - ceph.restart: daemons: - osd.8 - osd.9 - osd.10 - osd.11 - rgw.* wait-for-healthy: false wait-for-osds-up: true - print: '**** done restarted/upgraded => osd.8, osd.9, osd.10, osd.11, rgw.*' - exec: osd.0: - ceph osd set pglog_hardlimit - ceph osd dump --format=json-pretty | grep "flags" - print: '**** done try to set pglog_hardlimit again, should succeed' - rados: clients: - client.0 ec_pool: true erasure_code_profile: crush-failure-domain: osd k: 3 m: 1 name: jerasure31profile plugin: jerasure technique: reed_sol_van objects: 50 op_weights: append: 100 copy_from: 50 delete: 50 read: 100 rmattr: 25 rollback: 50 setattr: 25 snap_create: 50 snap_remove: 50 write: 0 ops: 4000 write_append_excl: false teuthology_branch: master tube: smithi verbose: false worker_log: /home/teuthworker/archive/worker_logs/worker.smithi.2475 2020-01-27T18:37:37.364 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_master/qa; will attempt to use it 2020-01-27T18:37:37.364 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_master/qa/tasks 2020-01-27T18:37:37.365 INFO:teuthology.run_tasks:Running task internal.check_packages... 2020-01-27T18:37:37.366 INFO:teuthology.task.internal:Checking packages... 2020-01-27T18:37:37.379 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'basic' and ceph hash '9c19d0edb97c85432761f769828070f563587117' 2020-01-27T18:37:37.379 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2020-01-27T18:37:37.379 INFO:teuthology.packaging:ref: None 2020-01-27T18:37:37.379 INFO:teuthology.packaging:tag: None 2020-01-27T18:37:37.380 INFO:teuthology.packaging:branch: master 2020-01-27T18:37:37.380 INFO:teuthology.packaging:sha1: 9c19d0edb97c85432761f769828070f563587117 2020-01-27T18:37:37.380 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F7%2Fx86_64&ref=master 2020-01-27T18:37:38.641 INFO:teuthology.task.internal:Found packages for ceph version 15.0.0-9849.ge33f610 2020-01-27T18:37:38.641 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2020-01-27T18:37:38.651 INFO:teuthology.task.internal:no buildpackages task found 2020-01-27T18:37:38.651 INFO:teuthology.run_tasks:Running task internal.lock_machines... 2020-01-27T18:37:38.717 INFO:teuthology.task.internal.lock_machines:Locking machines... 2020-01-27T18:37:38.717 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/ 2020-01-27T18:37:38.769 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T18:37:48.779 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:37:58.798 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:38:08.815 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:38:18.834 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)... 2020-01-27T18:38:28.851 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)... 2020-01-27T18:38:38.867 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)... 2020-01-27T18:38:48.883 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)... 2020-01-27T18:38:58.899 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:39:08.918 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:39:18.937 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:39:28.955 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:39:38.975 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:39:48.993 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:39:59.012 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:40:09.027 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:40:19.045 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:40:29.055 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:40:39.068 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:40:49.087 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:40:59.107 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:41:09.123 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:41:19.136 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:41:29.154 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:41:39.171 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:41:49.182 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:41:59.200 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:42:09.219 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:42:19.235 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:42:29.245 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:42:39.263 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:42:49.275 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:42:59.294 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:43:09.313 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:43:19.331 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:43:29.342 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:43:39.356 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:43:49.374 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:43:59.385 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:44:09.403 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:44:19.416 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:44:29.431 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:44:39.447 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:44:49.463 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:44:59.481 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:45:10.549 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:45:20.564 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:45:30.579 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:45:40.595 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:45:50.613 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:46:00.631 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:46:10.647 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:46:20.662 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:46:30.682 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:46:40.704 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:46:50.720 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:47:00.738 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:47:10.746 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:47:20.763 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:47:30.774 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:47:40.791 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T18:47:50.808 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T18:48:00.827 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T18:48:10.845 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T18:48:20.859 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T18:48:30.874 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T18:48:40.892 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T18:48:50.909 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 6)... 2020-01-27T18:49:00.921 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:49:10.938 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:49:20.957 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:49:30.974 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:49:40.990 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:49:51.008 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:50:01.020 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:50:11.032 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:50:21.048 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:50:31.067 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:50:41.081 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:50:51.100 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:51:01.118 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:51:11.136 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:51:21.150 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:51:31.167 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:51:41.182 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T18:51:51.200 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:52:01.217 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:52:11.234 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:52:21.247 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:52:31.264 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:52:41.282 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:52:51.299 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:53:01.315 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:53:11.329 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:53:21.346 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:53:31.363 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:53:41.382 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:53:51.392 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:54:01.411 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:54:11.425 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T18:54:21.445 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T18:54:31.455 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T18:54:41.472 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T18:54:51.489 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T18:55:01.506 DEBUG:teuthology.lock.ops:lock_many request: {'count': 5, 'locked_by': 'scheduled_dzafman@ubuntu-1804', 'description': '/home/teuthworker/archive/dzafman-2020-01-25_01:21:34-upgrade:mimic-x-master-distro-basic-smithi/4711859', 'machine_type': 'smithi'} 2020-01-27T18:55:01.518 ERROR:teuthology.lock.ops:Insufficient nodes available to lock 5 smithi nodes. 2020-01-27T18:55:01.518 ERROR:teuthology.lock.ops:{"message": "only 3 nodes available"} 2020-01-27T18:55:01.518 INFO:teuthology.task.internal.lock_machines:0 smithi machines locked this try, 0/5 locked so far 2020-01-27T18:55:01.518 INFO:teuthology.task.internal.lock_machines:0 machines locked (0 new); need 5 more 2020-01-27T18:55:01.518 WARNING:teuthology.task.internal.lock_machines:Could not lock enough machines, waiting... 2020-01-27T18:55:11.527 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:55:21.545 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:55:31.555 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:55:41.570 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:55:51.582 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:56:01.595 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:56:11.612 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:56:21.631 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:56:31.647 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:56:41.662 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:56:51.671 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:57:01.686 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:57:13.080 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:57:23.093 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:57:33.114 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:57:43.135 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:57:53.152 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:58:03.173 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:58:13.190 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:58:23.203 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:58:33.218 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:58:43.233 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:58:53.249 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:59:03.266 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:59:13.284 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:59:23.302 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:59:33.312 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T18:59:43.321 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)... 2020-01-27T18:59:53.339 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:00:03.352 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:00:13.370 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:00:23.387 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:00:33.404 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:00:43.420 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:00:53.434 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:01:03.445 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:01:13.462 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:01:23.480 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:01:33.498 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:01:43.513 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:01:53.533 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:02:03.542 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:02:13.558 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:02:23.574 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:02:33.592 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:02:43.611 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:02:53.630 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:03:03.648 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:03:13.658 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:03:23.669 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:03:33.687 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:03:43.705 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)... 2020-01-27T19:03:53.722 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:04:03.740 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:04:13.760 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:04:23.778 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:04:33.794 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:04:43.811 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:04:53.826 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:05:03.845 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:05:13.862 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:05:23.874 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:05:33.890 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:05:43.908 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:05:53.922 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:06:03.940 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:06:13.957 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:06:23.975 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:06:33.990 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:06:44.007 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:06:54.026 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:07:04.044 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:07:14.060 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:07:24.078 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:07:34.094 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:07:44.110 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:07:54.125 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:08:04.144 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:08:14.158 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:08:24.172 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:08:34.190 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:08:44.207 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:08:54.217 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:09:04.228 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:09:14.245 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:09:24.258 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:09:34.275 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:09:44.291 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:09:54.307 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:10:04.321 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:10:14.339 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:10:24.351 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:10:34.362 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:10:44.378 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:10:54.390 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:11:04.406 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:11:14.420 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:11:24.438 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:11:34.454 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:11:44.477 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:11:54.486 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:12:04.503 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:12:14.535 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:12:24.550 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:12:34.565 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:12:44.581 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:12:54.596 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:13:04.614 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:13:14.628 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:13:24.642 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:13:34.660 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:13:44.679 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:13:54.697 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:14:04.715 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:14:14.731 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:14:24.749 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:14:34.947 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:14:44.956 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:14:54.973 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:15:04.981 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:15:14.998 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:15:25.014 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:15:35.024 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:15:45.037 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:15:55.054 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T19:16:05.070 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:16:15.089 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:16:25.134 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:16:35.149 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:16:45.166 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:16:55.183 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:17:05.206 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:17:15.221 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:17:25.237 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:17:35.255 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:17:45.270 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:17:55.287 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:18:05.301 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:18:15.310 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:18:25.325 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:18:35.334 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:18:45.350 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T19:18:55.368 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)... 2020-01-27T19:19:05.383 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)... 2020-01-27T19:19:15.398 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:19:25.415 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:19:35.430 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:19:45.445 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:19:55.461 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:20:05.471 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:20:15.486 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:20:25.505 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:20:35.522 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:20:45.539 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:20:55.556 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:21:05.566 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:21:15.583 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:21:25.598 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:21:35.616 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:21:45.633 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:21:55.651 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:22:05.668 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)... 2020-01-27T19:22:15.686 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:22:25.701 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:22:35.718 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:22:45.735 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:22:55.753 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:23:05.769 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:23:15.785 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:23:25.802 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:23:35.819 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:23:45.836 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:23:55.846 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:24:05.862 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:24:15.879 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:24:25.897 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:24:35.914 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:24:45.929 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:24:55.947 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:25:05.956 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:25:15.970 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:25:25.980 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:25:35.994 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:25:46.011 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:25:56.027 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:26:06.040 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:26:16.053 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:26:26.069 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:26:36.085 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:26:46.095 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:26:56.107 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:27:06.125 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:27:16.145 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:27:26.158 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:27:36.175 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:27:46.190 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:27:56.207 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:28:06.224 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:28:16.234 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)... 2020-01-27T19:28:26.251 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...