2020-01-27T18:34:27.797 INFO:root:teuthology version: 1.0.0-7063d1e
2020-01-27T18:34:27.797 DEBUG:teuthology.run:Teuthology command: teuthology /tmp/teuthology-worker.FTHMMp.tmp -- --verbose --lock --description upgrade:mimic-x/stress-split/{0-cluster/{openstack.yaml start.yaml} 1-ceph-install/mimic.yaml 1.1-pg-log-overrides/short_pg_log.yaml 2-partial-upgrade/firsthalf.yaml 3-thrash/default.yaml 4-workload/{radosbench.yaml rbd-cls.yaml rbd-import-export.yaml rbd_api.yaml readwrite.yaml rgw_ragweed_prepare.yaml snaps-few-objects.yaml} 5-finish-upgrade.yaml 6-msgr2.yaml 6-nautilus.yaml 7-final-workload/{rbd-python.yaml rgw-swift-ragweed_check.yaml snaps-many-objects.yaml} objectstore/bluestore-bitmap.yaml supported-all-distro/centos_7.yaml thrashosds-health.yaml} --name dzafman-2020-01-25_01:21:34-upgrade:mimic-x-master-distro-basic-smithi --block --owner scheduled_dzafman@ubuntu-1804 --archive /home/teuthworker/archive/dzafman-2020-01-25_01:21:34-upgrade:mimic-x-master-distro-basic-smithi/4711855
2020-01-27T18:34:27.884 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-01-27T18:34:27.969 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/dzafman-2020-01-25_01:21:34-upgrade:mimic-x-master-distro-basic-smithi/4711855
  branch: master
  description: upgrade:mimic-x/stress-split/{0-cluster/{openstack.yaml start.yaml} 1-ceph-install/mimic.yaml
    1.1-pg-log-overrides/short_pg_log.yaml 2-partial-upgrade/firsthalf.yaml 3-thrash/default.yaml
    4-workload/{radosbench.yaml rbd-cls.yaml rbd-import-export.yaml rbd_api.yaml readwrite.yaml
    rgw_ragweed_prepare.yaml snaps-few-objects.yaml} 5-finish-upgrade.yaml 6-msgr2.yaml
    6-nautilus.yaml 7-final-workload/{rbd-python.yaml rgw-swift-ragweed_check.yaml snaps-many-objects.yaml}
    objectstore/bluestore-bitmap.yaml supported-all-distro/centos_7.yaml thrashosds-health.yaml}
  email: dzafman@redhat.com
  first_in_suite: false
  job_id: '4711855'
  kernel:
    kdb: true
    sha1: distro
  last_in_suite: false
  machine_type: smithi
  meta:
  - desc: 'Run ceph on two nodes,
  
      with a separate client-only node.
  
      Use xfs beneath the osds.
  
      #Note-To enable RHEL runs on ovh nodes, add the following to overrides
  
      #ansible.cephlab:
  
      # skip_tags: entitlements,packages,repos
  
      '
  - desc: install ceph/mimic latest
  - desc: 'install upgrade ceph/-x on 2/3 of cluster
  
      restart : mons, osd 0-7
  
      '
  - desc: 'randomly kill and revive osd
  
      small chance to increase the number of pgs
  
      '
  - desc: 'run randomized correctness test for rados operations
  
      generate write load with rados bench
  
      '
  - desc: 'run basic cls tests for rbd
  
      '
  - desc: 'run basic import/export cli tests for rbd
  
      '
  - desc: 'librbd C and C++ api tests
  
      '
  - desc: 'randomized correctness test for rados operations on a replicated pool,
  
      using only reads, writes, and deletes
  
      '
  - desc: 'rgw ragweed prepare before upgrade
  
      '
  - desc: 'randomized correctness test for rados operations on a replicated pool with
      snapshot operations
  
      '
  - desc: 'librbd python api tests
  
      '
  - desc: 'swift api tests for rgw
  
      rgw ragweed check after upgrade
  
      '
  - desc: 'randomized correctness test for rados operations on a replicated pool with
      snapshot operations
  
      '
  name: dzafman-2020-01-25_01:21:34-upgrade:mimic-x-master-distro-basic-smithi
  nuke-on-error: true
  openstack:
  - machine:
      disk: 100
  - volumes:
      count: 4
      size: 30
  os_type: centos
  os_version: '7.6'
  overrides:
    admin_socket:
      branch: master
    ceph:
      conf:
        global:
          enable experimental unrecoverable data corrupting features: '*'
          mon warn on msgr2 not enabled: false
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
          mon warn on osd down out interval zero: false
        osd:
          bdev async discard: true
          bdev enable discard: true
          bluestore allocator: bitmap
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          debug bluefs: 20
          debug bluestore: 20
          debug filestore: 20
          debug journal: 20
          debug ms: 1
          debug osd: 25
          debug rocksdb: 10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd failsafe full ratio: 0.95
          osd max pg log entries: 2
          osd min pg log entries: 1
          osd objectstore: bluestore
      fs: xfs
      log-whitelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      - overall HEALTH_
      - \(MON_DOWN\)
      - \(MGR_DOWN\)
      - \(MON_MSGR2_NOT_ENABLED\)
      - but it is still running
      - wrongly marked me down
      - objects unfound and apparently lost
      - log bound mismatch
      - overall HEALTH_
      - \(OSDMAP_FLAGS\)
      - \(OSD_
      - \(PG_
      - \(POOL_
      - \(CACHE_POOL_
      - \(SMALLER_PGP_NUM\)
      - \(OBJECT_
      - \(SLOW_OPS\)
      - \(REQUEST_SLOW\)
      - \(TOO_FEW_PGS\)
      - slow requests
      mon_bind_addrvec: false
      mon_bind_msgr2: false
      sha1: 9c19d0edb97c85432761f769828070f563587117
    ceph-deploy:
      bluestore: true
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon:
          osd default pool size: 2
        osd:
          bdev async discard: true
          bdev enable discard: true
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          debug bluefs: 20
          debug bluestore: 20
          debug rocksdb: 10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd failsafe full ratio: 0.95
          osd objectstore: bluestore
      fs: xfs
    install:
      ceph:
        sha1: 9c19d0edb97c85432761f769828070f563587117
    thrashosds:
      bdev_inject_crash: 2
      bdev_inject_crash_probability: 0.5
    workunit:
      branch: master
      sha1: 35b0b52153206fd6178afe584a2cd9c0ef2bcc05
  owner: scheduled_dzafman@ubuntu-1804
  priority: 50
  repo: git://git.ceph.com/ceph-ci.git
  roles:
  - - mon.a
    - mgr.x
    - osd.0
    - osd.1
    - osd.2
    - osd.3
  - - mon.b
    - osd.4
    - osd.5
    - osd.6
    - osd.7
  - - mon.c
  - - osd.8
    - osd.9
    - osd.10
    - osd.11
  - - client.0
  sha1: 9c19d0edb97c85432761f769828070f563587117
  stress-tasks:
  - thrashosds:
      aggressive_pg_num_changes: false
      chance_force_recovery: 0
      chance_pgnum_grow: 1
      chance_pgpnum_fix: 1
      chance_thrash_cluster_full: 0
      chance_thrash_pg_upmap: 0
      chance_thrash_pg_upmap_items: 0
      disable_objectstore_tool_tests: true
      timeout: 1200
  - print: '**** done thrashosds 3-thrash'
  - full_sequential:
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
  - print: '**** done radosbench 7-workload'
  - workunit:
      branch: mimic
      clients:
        client.0:
        - cls/test_cls_rbd.sh
  - print: '**** done cls/test_cls_rbd.sh 5-workload'
  - workunit:
      branch: mimic
      clients:
        client.0:
        - rbd/import_export.sh
      env:
        RBD_CREATE_ARGS: --new-format
  - print: '**** done rbd/import_export.sh 5-workload'
  - workunit:
      branch: mimic
      clients:
        client.0:
        - rbd/test_librbd.sh
  - print: '**** done rbd/test_librbd.sh 7-workload'
  - full_sequential:
    - rados:
        clients:
        - client.0
        objects: 500
        op_weights:
          delete: 10
          read: 45
          write: 45
        ops: 4000
        write_append_excl: false
  - print: '**** done rados/readwrite 5-workload'
  - full_sequential:
    - sequential:
      - ragweed:
          client.0:
            default-branch: ceph-nautilus
            rgw_server: client.0
            stages: prepare
      - print: '**** done rgw ragweed prepare 4-workload'
  - full_sequential:
    - rados:
        clients:
        - client.0
        objects: 50
        op_weights:
          delete: 50
          read: 100
          rollback: 50
          snap_create: 50
          snap_remove: 50
          write: 100
        ops: 4000
        write_append_excl: false
  - print: '**** done rados/snaps-few-objects 5-workload'
  suite: upgrade:mimic-x
  suite_branch: master
  suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_master/qa
  suite_relpath: qa
  suite_repo: git://git.ceph.com/ceph-ci.git
  suite_sha1: 35b0b52153206fd6178afe584a2cd9c0ef2bcc05
  tasks:
  - install:
      branch: mimic
      exclude_packages:
      - librados3
      - ceph-mgr-dashboard
      - ceph-mgr-diskprediction-local
      - ceph-mgr-diskprediction-cloud
      - ceph-mgr-rook
      - ceph-mgr-ssh
      - ceph-daemon
      extra_packages:
      - librados2
  - print: '**** done install mimic'
  - ceph:
      conf:
        global:
          bluestore warn on no per pool omap: false
          bluestore_warn_on_legacy_statfs: false
  - exec:
      osd.0:
      - ceph osd require-osd-release mimic
      - ceph osd set-require-min-compat-client mimic
  - print: '**** done ceph'
  - rgw:
    - client.0
  - print: '**** done => started rgw client.0'
  - install.upgrade:
      mon.a: null
      mon.b: null
      mon.c: null
  - print: '**** done install.upgrade first 3 nodes'
  - ceph.restart:
      daemons:
      - mon.a
      - mon.b
      - mon.c
      - mgr.x
      - osd.0
      - osd.1
      - osd.2
      - osd.3
      - osd.4
      - osd.5
      - osd.6
      - osd.7
      mon-health-to-clog: false
  - print: '**** done ceph.restart all mons and 2/3 of osds'
  - parallel:
    - stress-tasks
  - install.upgrade:
      client.0: null
      osd.8: null
  - ceph.restart:
      daemons:
      - osd.8
      - osd.9
      - osd.10
      - osd.11
      - rgw.*
      wait-for-healthy: false
      wait-for-osds-up: true
  - print: '**** done restarted/upgraded => osd.8, osd.9, osd.10, osd.11, rgw.*'
  - exec:
      osd.0:
      - ceph osd set pglog_hardlimit
      - ceph osd dump --format=json-pretty | grep "flags"
  - print: '**** done try to set pglog_hardlimit again, should succeed'
  - exec:
      mon.a:
      - ceph mon enable-msgr2
  - ceph.healthy: null
  - exec:
      osd.0:
      - ceph osd require-osd-release nautilus
      - ceph osd set-require-min-compat-client nautilus
      - for p in `ceph osd pool ls`; do ceph osd pool set $p pg_autoscale_mode off;
        done
  - ceph.healthy: null
  - workunit:
      branch: mimic
      clients:
        client.0:
        - rbd/test_librbd_python.sh
  - print: '**** done rbd/test_librbd_python.sh 9-workload'
  - swift:
      client.0:
        rgw_server: client.0
  - print: '**** done swift 7-workload'
  - ragweed:
      client.0:
        default-branch: ceph-nautilus
        rgw_server: client.0
        stages: check
  - print: '**** done rgw ragweed check 7-workload'
  - rados:
      clients:
      - client.0
      objects: 500
      op_weights:
        delete: 50
        read: 100
        rollback: 50
        snap_create: 50
        snap_remove: 50
        write: 100
      ops: 4000
      write_append_excl: false
  teuthology_branch: master
  tube: smithi
  verbose: false
  worker_log: /home/teuthworker/archive/worker_logs/worker.smithi.5524
2020-01-27T18:34:27.969 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_master/qa; will attempt to use it
2020-01-27T18:34:27.970 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_master/qa/tasks
2020-01-27T18:34:27.970 INFO:teuthology.run_tasks:Running task internal.check_packages...
2020-01-27T18:34:27.971 INFO:teuthology.task.internal:Checking packages...
2020-01-27T18:34:27.983 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'basic' and ceph hash '9c19d0edb97c85432761f769828070f563587117'
2020-01-27T18:34:27.983 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2020-01-27T18:34:27.984 INFO:teuthology.packaging:ref: None
2020-01-27T18:34:27.984 INFO:teuthology.packaging:tag: None
2020-01-27T18:34:27.984 INFO:teuthology.packaging:branch: master
2020-01-27T18:34:27.984 INFO:teuthology.packaging:sha1: 9c19d0edb97c85432761f769828070f563587117
2020-01-27T18:34:27.984 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F7%2Fx86_64&ref=master
2020-01-27T18:34:29.304 INFO:teuthology.task.internal:Found packages for ceph version 15.0.0-9849.ge33f610
2020-01-27T18:34:29.304 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2020-01-27T18:34:29.314 INFO:teuthology.task.internal:no buildpackages task found
2020-01-27T18:34:29.314 INFO:teuthology.run_tasks:Running task internal.lock_machines...
2020-01-27T18:34:29.327 INFO:teuthology.task.internal.lock_machines:Locking machines...
2020-01-27T18:34:29.328 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-01-27T18:34:29.390 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:34:39.409 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 6)...
2020-01-27T18:34:49.427 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:34:59.444 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:35:09.463 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:35:19.480 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:35:29.500 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:35:39.535 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:35:49.554 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:35:59.573 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:36:09.590 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:36:19.607 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:36:29.622 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:36:39.633 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:36:49.652 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:36:59.668 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:37:09.684 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:37:19.699 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:37:29.715 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:37:39.731 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:37:49.748 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:37:59.770 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:38:09.798 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:38:19.815 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)...
2020-01-27T18:38:29.834 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)...
2020-01-27T18:38:39.853 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)...
2020-01-27T18:38:49.871 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)...
2020-01-27T18:38:59.890 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:39:09.909 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:39:19.928 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:39:29.947 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:39:39.957 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:39:49.975 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:39:59.993 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:40:10.004 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:40:20.019 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:40:30.038 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:40:40.057 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:40:50.076 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:41:00.095 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:41:10.107 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:41:20.122 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:41:30.143 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:41:40.159 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:41:50.178 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:42:00.198 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:42:10.215 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:42:20.227 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:42:30.246 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:42:40.262 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:42:50.277 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:43:00.296 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:43:10.315 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:43:20.330 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:43:30.349 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:43:40.368 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:43:50.383 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:44:00.402 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:44:10.419 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:44:20.438 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:44:30.456 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:44:40.469 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:44:50.481 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:45:00.500 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:45:10.557 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:45:20.569 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:45:30.586 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:45:40.602 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:45:50.612 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:46:00.631 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:46:10.648 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:46:20.662 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:46:30.679 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:46:40.695 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:46:50.709 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:47:00.731 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:47:10.742 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:47:20.761 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:47:30.778 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:47:40.795 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:47:50.810 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:48:00.827 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:48:10.844 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:48:20.859 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:48:30.868 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:48:40.886 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:48:50.899 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 6)...
2020-01-27T18:49:00.912 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:49:10.930 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:49:20.946 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:49:30.964 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:49:40.978 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:49:50.994 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:50:01.013 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:50:11.031 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:50:21.049 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:50:31.068 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:50:41.081 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:50:51.093 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:51:01.111 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:51:11.136 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:51:21.155 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:51:31.173 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:51:41.196 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:51:51.213 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:52:01.231 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:52:11.248 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:52:21.262 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:52:31.280 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:52:41.295 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:52:51.314 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:53:01.323 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:53:11.335 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:53:21.350 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:53:31.367 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:53:41.382 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:53:51.397 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:54:01.414 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:54:11.426 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T18:54:21.436 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:54:31.454 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:54:41.472 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:54:51.482 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T18:55:01.499 DEBUG:teuthology.lock.ops:lock_many request: {'count': 5, 'locked_by': 'scheduled_dzafman@ubuntu-1804', 'description': '/home/teuthworker/archive/dzafman-2020-01-25_01:21:34-upgrade:mimic-x-master-distro-basic-smithi/4711855', 'machine_type': 'smithi'}
2020-01-27T18:55:01.514 ERROR:teuthology.lock.ops:Insufficient nodes available to lock 5 smithi nodes.
2020-01-27T18:55:01.515 ERROR:teuthology.lock.ops:{"message": "only 3 nodes available"}
2020-01-27T18:55:01.515 INFO:teuthology.task.internal.lock_machines:0 smithi machines locked this try, 0/5 locked so far
2020-01-27T18:55:01.515 INFO:teuthology.task.internal.lock_machines:0 machines locked (0 new); need 5 more
2020-01-27T18:55:01.515 WARNING:teuthology.task.internal.lock_machines:Could not lock enough machines, waiting...
2020-01-27T18:55:11.527 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:55:21.541 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:55:31.551 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:55:41.570 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:55:51.586 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:56:01.605 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:56:11.623 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:56:21.640 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:56:31.658 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:56:41.676 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:56:51.696 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:57:01.713 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:57:13.084 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:57:23.102 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:57:33.118 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:57:43.135 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:57:53.152 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:58:03.164 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:58:13.182 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:58:23.193 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:58:33.211 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:58:43.226 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:58:53.244 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:59:03.255 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:59:13.274 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:59:23.291 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:59:33.309 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T18:59:43.326 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)...
2020-01-27T18:59:53.344 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:00:03.357 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:00:13.368 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:00:23.387 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:00:33.402 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:00:43.420 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:00:53.434 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:01:03.452 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:01:13.467 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:01:23.485 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:01:33.502 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:01:43.520 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:01:53.531 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:02:03.549 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:02:13.566 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:02:23.582 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:02:33.600 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:02:43.611 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:02:53.626 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:03:03.636 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:03:13.649 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:03:23.664 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:03:33.681 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:03:43.692 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)...
2020-01-27T19:03:53.711 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:04:03.730 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:04:13.746 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:04:23.757 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:04:33.773 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:04:43.785 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:04:53.803 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:05:03.817 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:05:13.831 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:05:23.847 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:05:33.860 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:05:43.875 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:05:53.894 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:06:03.912 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:06:13.923 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:06:23.935 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:06:33.954 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:06:43.969 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:06:53.978 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:07:03.997 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:07:14.010 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:07:24.029 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:07:34.046 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:07:44.062 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:07:54.078 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:08:04.095 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:08:14.110 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:08:24.127 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:08:34.143 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:08:44.159 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:08:54.176 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:09:04.199 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:09:14.208 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:09:24.224 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:09:34.239 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:09:44.255 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:09:54.273 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:10:04.281 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:10:14.298 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:10:24.316 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:10:34.330 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:10:44.340 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:10:54.349 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:11:04.361 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:11:14.379 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:11:24.397 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:11:34.411 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:11:44.424 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:11:54.433 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:12:04.451 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:12:14.489 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:12:24.647 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:12:34.663 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:12:44.674 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:12:54.684 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:13:04.697 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:13:14.714 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:13:24.726 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:13:34.738 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:13:44.754 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:13:54.770 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:14:04.787 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:14:14.803 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:14:24.823 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:14:34.838 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:14:44.853 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:14:54.870 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:15:04.887 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:15:14.903 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:15:24.914 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:15:34.930 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:15:44.947 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:15:54.964 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T19:16:04.977 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:16:14.998 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:16:25.113 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:16:35.130 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:16:45.143 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:16:55.159 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:17:05.183 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:17:15.200 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:17:25.218 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:17:35.235 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:17:45.251 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:17:55.269 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:18:05.286 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:18:15.299 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:18:25.315 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:18:35.330 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:18:45.340 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T19:18:55.350 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 4)...
2020-01-27T19:19:05.365 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 5)...
2020-01-27T19:19:15.382 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:19:25.400 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:19:35.418 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:19:45.434 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:19:55.452 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:20:05.478 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:20:15.494 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:20:25.509 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:20:35.526 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:20:45.542 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:20:55.559 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:21:05.574 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:21:15.586 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:21:25.605 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:21:35.622 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:21:45.632 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:21:55.651 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:22:05.665 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 3)...
2020-01-27T19:22:15.678 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:22:25.695 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:22:35.711 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:22:45.726 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:22:55.743 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:23:05.761 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:23:15.770 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:23:25.787 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:23:35.799 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:23:45.816 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:23:55.832 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:24:05.853 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:24:15.868 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:24:25.878 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:24:35.896 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:24:45.910 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:24:55.927 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:25:05.942 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:25:15.959 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:25:25.977 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:25:35.995 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:25:46.012 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:25:56.021 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:26:06.040 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:26:16.051 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:26:26.067 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:26:36.078 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:26:46.091 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:26:56.107 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:27:06.125 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:27:16.149 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:27:26.166 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:27:36.183 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:27:46.197 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:27:56.207 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:28:06.223 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:28:16.235 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...
2020-01-27T19:28:26.250 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 5, have 2)...