2020-03-27T15:20:26.765 INFO:root:teuthology version: 1.0.0-b3a6b1a
2020-03-27T15:20:26.766 DEBUG:teuthology.run:Teuthology command: teuthology /tmp/teuthology-worker.HULCiI.tmp -- --verbose --lock --description rados:thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-install/luminous.yaml backoff/peering.yaml ceph.yaml clusters/{openstack.yaml three-plus-one.yaml} d-balancer/crush-compat.yaml distro$/{centos_7.6.yaml} msgr-failures/osd-delay.yaml rados.yaml thrashers/pggrow.yaml thrashosds-health.yaml workloads/radosbench.yaml} --name sage-2020-03-27_15:12:41-rados:thrash-old-clients-revert-dec-distro-basic-smithi --block --owner scheduled_sage@teuthology --archive /home/teuthworker/archive/sage-2020-03-27_15:12:41-rados:thrash-old-clients-revert-dec-distro-basic-smithi/4895504
2020-03-27T15:20:26.809 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-03-27T15:20:26.868 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/sage-2020-03-27_15:12:41-rados:thrash-old-clients-revert-dec-distro-basic-smithi/4895504
  branch: revert-dec
  description: rados:thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size.yaml
    1-install/luminous.yaml backoff/peering.yaml ceph.yaml clusters/{openstack.yaml
    three-plus-one.yaml} d-balancer/crush-compat.yaml distro$/{centos_7.6.yaml} msgr-failures/osd-delay.yaml
    rados.yaml thrashers/pggrow.yaml thrashosds-health.yaml workloads/radosbench.yaml}
  email: null
  first_in_suite: false
  job_id: '4895504'
  kernel:
    kdb: true
    sha1: distro
  last_in_suite: false
  machine_type: smithi
  name: sage-2020-03-27_15:12:41-rados:thrash-old-clients-revert-dec-distro-basic-smithi
  nuke-on-error: true
  openstack:
  - volumes:
      count: 4
      size: 30
  - volumes:
      count: 4
      size: 10
  os_type: centos
  os_version: '7.6'
  overrides:
    admin_socket:
      branch: revert-dec
    ceph:
      conf:
        client.2:
          debug ms: 1
          debug objecter: 20
          debug rados: 20
        global:
          mon client directed command retry: 5
          mon warn on msgr2 not enabled: false
          ms inject delay max: 1
          ms inject delay probability: 0.005
          ms inject delay type: osd
          ms inject internal delays: 0.002
          ms inject socket failures: 2500
          osd_pool_default_min_size: 2
          osd_pool_default_size: 2
        mgr:
          debug mgr: 20
          debug ms: 1
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
          mon min osdmap epochs: 50
          mon osdmap full prune interval: 2
          mon osdmap full prune min: 15
          mon osdmap full prune txsize: 2
          paxos service trim min: 10
        osd:
          debug filestore: 20
          debug journal: 20
          debug ms: 20
          debug osd: 25
          filestore odsync write: true
          osd backoff on peering: true
          osd debug verify cached snaps: true
          osd debug verify missing on start: true
          osd max backfills: 2
          osd max markdown count: 1000
          osd op queue: debug_random
          osd op queue cut off: debug_random
          osd scrub max interval: 120
          osd scrub min interval: 60
          osd shutdown pgref assert: true
          osd snap trim sleep: 0.5
      log-whitelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      - \(REQUEST_STUCK\)
      - \(MON_DOWN\)
      - \(OSD_SLOW_PING_TIME
      - but it is still running
      - objects unfound and apparently lost
      - overall HEALTH_
      - \(OSDMAP_FLAGS\)
      - \(OSD_
      - \(PG_
      - \(POOL_
      - \(CACHE_POOL_
      - \(SMALLER_PGP_NUM\)
      - \(OBJECT_
      - \(SLOW_OPS\)
      - \(REQUEST_SLOW\)
      - \(TOO_FEW_PGS\)
      - slow request
      mon_bind_msgr2: false
      sha1: 04af5888e1368740ebb4403bab567f8cfd92df6c
    ceph-deploy:
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon:
          osd default pool size: 2
    install:
      ceph:
        sha1: 04af5888e1368740ebb4403bab567f8cfd92df6c
    selinux:
      whitelist:
      - scontext=system_u:system_r:logrotate_t:s0
    workunit:
      branch: revert-dec
      sha1: 04af5888e1368740ebb4403bab567f8cfd92df6c
  owner: scheduled_sage@teuthology
  priority: 50
  repo: git://git.ceph.com/ceph-ci.git
  roles:
  - - mon.a
    - mgr.y
    - osd.0
    - osd.1
    - osd.2
    - osd.3
    - client.0
  - - mon.b
    - mgr.x
    - osd.4
    - osd.5
    - osd.6
    - osd.7
    - client.1
  - - mon.c
    - osd.8
    - osd.9
    - osd.10
    - osd.11
    - client.2
  sha1: 04af5888e1368740ebb4403bab567f8cfd92df6c
  suite: rados:thrash-old-clients
  suite_branch: revert-dec
  suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_revert-dec/qa
  suite_relpath: qa
  suite_repo: git://git.ceph.com/ceph-ci.git
  suite_sha1: 04af5888e1368740ebb4403bab567f8cfd92df6c
  tasks:
  - install:
      branch: luminous
      exclude_packages:
      - librados3
      - ceph-mgr-dashboard
      - ceph-mgr-diskprediction-local
      - ceph-mgr-rook
      - ceph-mgr-cephadm
      - cephadm
      - ceph-immutable-object-cache
      - ceph-base
      - python3-rados
      - python3-rgw
      - python3-rbd
      - python3-cephfs
      - librados-devel
      extra_packages:
      - librados2
      - python-rados
      - python-rgw
      - python-rbd
      - python-cephfs
  - cephadm: null
  - exec:
      mon.a:
      - while ! ceph balancer status ; do sleep 1 ; done
      - ceph balancer mode crush-compat
      - ceph balancer on
  - thrashosds:
      chance_pgnum_grow: 2
      chance_pgpnum_fix: 1
      timeout: 1200
  - full_sequential:
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
  teuthology_branch: master
  tube: smithi
  verbose: false
  worker_log: /home/teuthworker/archive/worker_logs/worker.smithi.27861
2020-03-27T15:20:26.868 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_revert-dec/qa; will attempt to use it
2020-03-27T15:20:26.869 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_revert-dec/qa/tasks
2020-03-27T15:20:26.869 INFO:teuthology.run_tasks:Running task internal.check_packages...
2020-03-27T15:20:26.871 INFO:teuthology.task.internal:Checking packages...
2020-03-27T15:20:26.883 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'basic' and ceph hash '04af5888e1368740ebb4403bab567f8cfd92df6c'
2020-03-27T15:20:26.884 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2020-03-27T15:20:26.884 INFO:teuthology.packaging:ref: None
2020-03-27T15:20:26.884 INFO:teuthology.packaging:tag: None
2020-03-27T15:20:26.884 INFO:teuthology.packaging:branch: revert-dec
2020-03-27T15:20:26.884 INFO:teuthology.packaging:sha1: 04af5888e1368740ebb4403bab567f8cfd92df6c
2020-03-27T15:20:26.884 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F7%2Fx86_64&ref=revert-dec
2020-03-27T15:20:27.116 INFO:teuthology.task.internal:Found packages for ceph version 15.2.0-377.g04af588
2020-03-27T15:20:27.116 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2020-03-27T15:20:27.130 INFO:teuthology.task.internal:no buildpackages task found
2020-03-27T15:20:27.130 INFO:teuthology.run_tasks:Running task internal.lock_machines...
2020-03-27T15:20:27.278 INFO:teuthology.task.internal.lock_machines:Locking machines...
2020-03-27T15:20:27.278 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-03-27T15:20:27.328 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:20:37.344 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:20:47.360 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:20:57.377 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:21:07.392 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:21:17.408 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:21:27.424 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:21:37.440 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:21:47.457 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:21:57.473 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:22:07.489 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:22:17.507 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:22:27.552 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:22:37.611 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:22:47.635 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:22:57.644 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:23:07.667 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:23:17.683 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:23:27.700 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:23:37.724 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:23:47.743 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:23:57.756 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:24:07.773 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:24:17.789 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:24:27.805 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:24:37.821 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:24:47.838 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:24:57.851 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:25:07.866 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:25:17.877 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:25:27.903 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:25:37.921 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:25:47.937 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:25:57.954 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:26:07.974 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:26:17.991 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:26:28.009 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:26:38.026 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:26:48.045 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:26:58.069 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:27:08.081 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:27:18.096 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:27:28.119 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:27:38.135 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:27:48.153 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:27:58.168 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:28:08.185 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:28:18.201 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:28:28.214 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:28:38.230 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:28:48.248 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:28:58.263 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:29:08.287 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:29:18.300 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:29:28.317 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:29:38.332 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:29:48.348 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:29:58.364 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:30:08.381 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:30:18.398 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:30:28.415 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:30:38.426 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:30:48.446 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:30:58.456 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:31:08.474 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:31:18.490 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:31:28.505 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:31:38.522 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:31:48.540 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:31:58.556 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:32:08.570 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:32:18.579 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:32:28.596 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:32:38.613 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:32:48.621 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:32:58.640 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:33:08.666 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:33:18.682 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:33:28.701 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:33:38.718 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:33:48.730 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:33:58.747 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:34:08.756 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:34:18.773 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:34:28.791 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:34:38.809 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:34:48.824 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:34:58.842 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:35:08.856 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:35:18.873 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:35:28.889 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:35:38.907 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:35:48.920 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:35:58.937 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:36:08.947 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:36:18.966 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:36:28.978 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:36:38.994 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:36:49.007 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:36:59.021 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:37:09.035 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:37:19.045 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:37:29.054 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:37:39.070 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:37:49.082 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:37:59.096 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:38:09.105 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:38:19.122 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:38:29.143 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:38:39.160 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:38:49.177 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:38:59.194 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:39:09.210 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:39:19.233 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:39:29.249 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:39:39.269 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:39:49.284 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:39:59.293 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:40:09.313 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:40:19.327 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:40:29.342 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:40:39.357 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:40:49.373 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:40:59.392 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:41:09.410 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:41:19.425 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:41:29.441 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:41:39.457 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:41:49.473 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:41:59.491 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:42:09.505 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:42:19.522 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:42:29.538 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:42:39.549 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:42:49.560 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:42:59.577 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:43:09.592 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:43:19.611 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:43:29.625 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:43:39.643 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:43:49.654 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:43:59.670 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:44:09.687 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:44:19.707 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:44:29.725 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:44:39.741 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:44:49.777 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:44:59.796 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:45:09.815 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:45:19.829 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:45:29.848 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:45:39.866 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:45:49.882 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:45:59.897 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:46:09.913 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:46:19.930 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:46:29.945 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:46:39.963 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:46:49.981 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:46:59.999 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:47:10.017 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:47:20.033 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:47:30.053 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:47:40.078 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:47:50.092 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:48:00.108 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:48:10.121 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:48:20.137 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:48:30.154 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:48:40.171 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:48:50.185 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:49:00.202 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:49:10.217 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:49:20.231 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:49:30.244 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:49:40.262 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:49:50.279 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:50:00.296 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:50:10.315 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:50:20.330 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:50:30.345 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:50:40.362 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T15:50:50.373 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:51:00.390 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:51:10.407 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:51:20.424 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:51:30.441 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:51:40.458 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:51:50.467 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:52:00.477 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:52:10.495 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:52:20.508 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:52:30.526 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:52:40.540 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:52:50.554 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:53:00.570 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:53:10.587 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:53:20.605 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:53:30.621 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:53:40.640 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:53:50.655 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:54:00.675 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:54:10.689 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:54:20.710 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:54:30.728 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:54:40.745 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:54:50.764 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:55:00.781 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:55:10.796 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:55:20.814 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:55:30.832 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:55:40.849 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:55:50.865 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:56:00.898 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:56:10.914 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:56:20.931 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:56:30.949 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:56:40.968 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:56:50.985 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:57:01.001 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:57:11.017 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:57:21.033 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:57:31.051 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:57:41.067 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:57:51.083 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T15:58:01.098 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:58:11.113 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:58:21.123 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:58:31.141 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:58:41.158 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:58:51.173 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:59:01.190 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:59:11.205 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:59:21.223 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:59:31.237 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:59:41.253 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T15:59:51.268 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T16:00:01.284 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T16:00:11.300 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...