2020-08-31T19:05:29.685 INFO:root:teuthology version: 1.0.0-1d15e8d
2020-08-31T19:05:29.687 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --name pdonnell-2020-08-31_18:14:29-kcephfs-wip-pdonnell-testing-20200829.172552-distro-basic-smithi --archive /home/teuthworker/archive/pdonnell-2020-08-31_18:14:29-kcephfs-wip-pdonnell-testing-20200829.172552-distro-basic-smithi/5396013 --lock --description kcephfs/recovery/{begin clusters/1-mds-4-client conf/{client mds mon osd} kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{frag_enable log-config osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/data-scan} --owner scheduled_pdonnell@teuthology --block -- /tmp/teuthology-worker.xauw4ams.tmp
2020-08-31T19:05:29.723 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-08-31T19:05:29.781 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/pdonnell-2020-08-31_18:14:29-kcephfs-wip-pdonnell-testing-20200829.172552-distro-basic-smithi/5396013
  branch: wip-pdonnell-testing-20200829.172552
  description: kcephfs/recovery/{begin clusters/1-mds-4-client conf/{client mds mon
    osd} kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}}
    objectstore-ec/bluestore-comp overrides/{frag_enable log-config osd-asserts whitelist_health
    whitelist_wrongly_marked_down} tasks/data-scan}
  email: pdonnell@redhat.com
  first_in_suite: false
  job_id: '5396013'
  kernel:
    client:
      branch: testing
    kdb: true
    sha1: distro
  last_in_suite: false
  log-rotate:
    ceph-mds: 10G
    ceph-osd: 10G
  machine_type: smithi
  name: pdonnell-2020-08-31_18:14:29-kcephfs-wip-pdonnell-testing-20200829.172552-distro-basic-smithi
  nuke-on-error: true
  openstack:
  - volumes:
      count: 4
      size: 30
  - machine:
      disk: 200
  os_type: ubuntu
  os_version: '18.04'
  overrides:
    admin_socket:
      branch: wip-pdonnell-testing-20200829.172552
    ceph:
      conf:
        client:
          client mount timeout: 600
          debug client: 20
          debug ms: 1
        global:
          ms die on skipped message: false
        mds:
          debug mds: 20
          debug ms: 1
          mds bal frag: true
          mds bal fragment size max: 10000
          mds bal merge size: 5
          mds bal split bits: 3
          mds bal split size: 100
          mds debug frag: true
          mds debug scatterstat: true
          mds op complaint time: 180
          mds verify scatter: true
          osd op complaint time: 180
        mgr:
          debug mgr: 20
          debug ms: 1
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
          mon op complaint time: 120
        osd:
          bluestore block size: 96636764160
          bluestore compression mode: aggressive
          bluestore fsck on mount: true
          debug bluefs: 20
          debug bluestore: 20
          debug filestore: 20
          debug journal: 20
          debug ms: 20
          debug osd: 25
          debug rocksdb: 10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd failsafe full ratio: 0.95
          osd objectstore: bluestore
          osd op complaint time: 180
          osd shutdown pgref assert: true
      fs: xfs
      log-ignorelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      - overall HEALTH_
      - \(FS_DEGRADED\)
      - \(MDS_FAILED\)
      - \(MDS_DEGRADED\)
      - \(FS_WITH_FAILED_MDS\)
      - \(MDS_DAMAGE\)
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      - \(FS_INLINE_DATA_DEPRECATED\)
      - overall HEALTH_
      - \(OSD_DOWN\)
      - \(OSD_
      - but it is still running
      - is not responding
      - bad backtrace
      - object missing on disk
      - error reading table object
      - error reading sessionmap
      - unmatched fragstat
      - was unreadable, recreating it now
      - Scrub error on inode
      - Metadata damage detected
      - inconsistent rstat on inode
      - Error recovering journal
      log-whitelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      sha1: c5af0d54b89ffb9f4c441f7a4e930a89dee8fecf
    ceph-deploy:
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon:
          osd default pool size: 2
    install:
      ceph:
        sha1: c5af0d54b89ffb9f4c441f7a4e930a89dee8fecf
    thrashosds:
      bdev_inject_crash: 2
      bdev_inject_crash_probability: 0.5
    workunit:
      branch: wip-pdonnell-testing-20200829.172552
      sha1: c5af0d54b89ffb9f4c441f7a4e930a89dee8fecf
  owner: scheduled_pdonnell@teuthology
  priority: 75
  repo: git://git.ceph.com/ceph-ci.git
  roles:
  - - mon.a
    - mgr.y
    - mds.a
    - mds.b
    - osd.0
    - osd.1
    - osd.2
    - osd.3
  - - mon.b
    - mon.c
    - mgr.x
    - mds.c
    - osd.4
    - osd.5
    - osd.6
    - osd.7
  - - client.0
  - - client.1
  - - client.2
  - - client.3
  sha1: c5af0d54b89ffb9f4c441f7a4e930a89dee8fecf
  sleep_before_teardown: 0
  suite: kcephfs
  suite_branch: wip-pdonnell-testing-20200829.172552
  suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_wip-pdonnell-testing-20200829.172552/qa
  suite_relpath: qa
  suite_repo: git://git.ceph.com/ceph-ci.git
  suite_sha1: c5af0d54b89ffb9f4c441f7a4e930a89dee8fecf
  tasks:
  - install:
      extra_packages:
        deb:
        - python3-cephfs
        - cephfs-shell
        rpm:
        - python3-cephfs
      extra_system_packages:
        deb:
        - bison
        - flex
        - libelf-dev
        - libssl-dev
        - network-manager
        - iproute2
        - util-linux
        - dump
        - indent
        - libaio-dev
        - libtool-bin
        - uuid-dev
        - xfslibs-dev
        rpm:
        - bison
        - flex
        - elfutils-libelf-devel
        - openssl-devel
        - NetworkManager
        - iproute
        - util-linux
        - libacl-devel
        - libaio-devel
        - libattr-devel
        - libtool
        - libuuid-devel
        - xfsdump
        - xfsprogs
        - xfsprogs-devel
        - libaio-devel
        - libtool
        - libuuid-devel
        - xfsprogs-devel
  - ceph: null
  - kclient: null
  - cephfs_test_runner:
      modules:
      - tasks.cephfs.test_data_scan
  teuthology_branch: rpm-test
  timestamp: 2020-08-31_18:14:29
  tube: smithi
  user: pdonnell
  verbose: false
  worker_log: /home/teuthworker/archive/worker_logs/worker.smithi.29917
2020-08-31T19:05:29.781 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_wip-pdonnell-testing-20200829.172552/qa; will attempt to use it
2020-08-31T19:05:29.782 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_wip-pdonnell-testing-20200829.172552/qa/tasks
2020-08-31T19:05:29.782 INFO:teuthology.run_tasks:Running task internal.check_packages...
2020-08-31T19:05:29.784 INFO:teuthology.task.internal:Checking packages...
2020-08-31T19:05:29.805 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'basic' and ceph hash 'c5af0d54b89ffb9f4c441f7a4e930a89dee8fecf'
2020-08-31T19:05:29.806 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2020-08-31T19:05:29.806 INFO:teuthology.packaging:ref: None
2020-08-31T19:05:29.807 INFO:teuthology.packaging:tag: None
2020-08-31T19:05:29.807 INFO:teuthology.packaging:branch: wip-pdonnell-testing-20200829.172552
2020-08-31T19:05:29.807 INFO:teuthology.packaging:sha1: c5af0d54b89ffb9f4c441f7a4e930a89dee8fecf
2020-08-31T19:05:29.808 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F18.04%2Fx86_64&ref=wip-pdonnell-testing-20200829.172552
2020-08-31T19:05:29.981 INFO:teuthology.task.internal:Found packages for ceph version 16.0.0-4885-gc5af0d5-1bionic
2020-08-31T19:05:29.981 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2020-08-31T19:05:29.989 INFO:teuthology.task.internal:no buildpackages task found
2020-08-31T19:05:29.989 INFO:teuthology.run_tasks:Running task internal.lock_machines...
2020-08-31T19:05:30.000 INFO:teuthology.task.internal.lock_machines:Locking machines...
2020-08-31T19:05:30.000 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-08-31T19:05:30.048 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:05:40.066 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:05:50.083 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:06:00.099 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:06:10.115 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:06:20.135 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:06:30.153 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:06:40.174 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:06:50.193 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:07:00.211 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:07:10.237 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:07:20.255 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:07:30.274 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:07:40.293 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:07:50.320 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:08:00.340 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:08:10.363 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:08:20.378 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:08:30.405 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:08:40.425 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:08:50.445 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:09:00.464 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:09:10.492 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:09:20.513 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:09:30.530 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:09:40.546 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:09:50.578 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:10:00.599 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:10:10.615 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:10:20.637 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:10:30.656 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:10:40.742 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:10:50.996 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:11:01.015 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:11:11.053 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:11:21.078 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:11:31.094 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:11:41.112 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:11:51.139 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:12:01.159 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:12:11.175 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 4)...
2020-08-31T19:12:21.194 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:12:31.209 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:12:41.232 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:12:51.252 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:13:01.271 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:13:11.297 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:13:21.324 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:13:31.343 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:13:41.359 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:13:51.377 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:14:01.400 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:14:11.422 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:14:21.434 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:14:31.454 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:14:41.499 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:14:51.518 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:15:01.538 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:15:11.555 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:15:21.608 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:15:31.623 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:15:41.639 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:15:51.662 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:16:01.692 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:16:11.731 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:16:21.758 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:16:31.778 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 3)...
2020-08-31T19:16:41.820 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:16:51.836 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:17:01.868 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:17:11.896 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:17:21.923 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:17:31.942 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:17:41.961 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:17:51.986 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:18:02.005 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:18:12.024 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:18:22.046 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:18:32.067 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:18:42.088 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:18:52.114 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:19:02.141 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:19:12.186 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:19:22.203 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:19:32.219 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:19:42.235 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:19:52.264 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:20:02.296 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:20:12.313 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:20:22.334 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:20:32.358 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:20:42.372 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 3)...
2020-08-31T19:20:52.399 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:21:02.415 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:21:12.446 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:21:22.463 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:21:32.481 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:21:42.498 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:21:52.547 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:22:02.567 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:22:12.595 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:22:22.612 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:22:32.635 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:22:42.651 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:22:52.669 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:23:02.704 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:23:12.736 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:23:22.751 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:23:32.766 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:23:42.791 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:23:52.804 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:24:02.823 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:24:12.842 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:24:22.863 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:24:32.887 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:24:42.913 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:24:52.930 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:25:02.945 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:25:12.962 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:25:23.021 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:25:33.038 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:25:43.054 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:25:53.075 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:26:03.098 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:26:13.113 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:26:23.129 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:26:33.146 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:26:43.167 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:26:53.182 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:27:03.198 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:27:13.215 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:27:23.242 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:27:33.258 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:27:43.287 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:27:53.323 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:28:03.339 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:28:13.374 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 5)...
2020-08-31T19:28:23.391 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:28:33.434 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 3)...
2020-08-31T19:28:43.460 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:28:53.477 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:29:03.496 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:29:13.521 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:29:23.539 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:29:33.559 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:29:43.585 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:29:53.602 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:30:03.619 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:30:13.637 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:30:23.663 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:30:33.686 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:30:43.708 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:30:53.763 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:31:03.794 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:31:13.822 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:31:23.849 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:31:33.873 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:31:43.889 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:31:53.909 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:32:03.946 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:32:13.963 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:32:23.982 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:32:33.998 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:32:44.015 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:32:54.030 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:33:04.049 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:33:14.072 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:33:24.098 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:33:34.117 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:33:44.134 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:33:54.165 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:34:04.181 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:34:14.205 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:34:24.227 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:34:34.252 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:34:44.271 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:34:54.288 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:35:04.307 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:35:14.323 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:35:24.347 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:35:34.359 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:35:44.375 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:35:54.390 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:36:04.412 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:36:14.433 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:36:24.455 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:36:34.470 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:36:44.485 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:36:54.513 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:37:04.532 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:37:14.549 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 3)...
2020-08-31T19:37:24.566 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 3)...
2020-08-31T19:37:34.589 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 3)...
2020-08-31T19:37:44.605 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 3)...
2020-08-31T19:37:54.623 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:38:04.640 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:38:14.658 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:38:24.679 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:38:34.706 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:38:44.726 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:38:54.744 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:39:04.765 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 2)...
2020-08-31T19:39:14.782 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:39:24.798 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:39:34.827 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:39:44.844 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:39:54.864 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:40:04.883 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:40:14.894 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:40:24.910 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:40:34.933 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:40:44.962 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:40:54.982 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:41:04.999 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:41:15.026 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:41:25.048 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:41:35.102 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:41:45.119 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:41:55.138 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:42:05.162 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:42:15.186 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:42:25.208 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 3)...
2020-08-31T19:42:35.236 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 3)...
2020-08-31T19:42:45.273 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:42:55.301 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:43:05.327 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:43:15.349 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:43:25.366 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:43:35.388 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:43:45.398 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:43:55.418 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:44:05.435 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:44:15.451 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 0)...
2020-08-31T19:44:25.466 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:44:35.484 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:44:45.503 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:44:55.517 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:45:05.542 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:45:15.561 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:45:25.583 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:45:35.599 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:45:45.619 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:45:55.638 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:46:05.656 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:46:15.673 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...
2020-08-31T19:46:25.689 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 6, have 1)...