2020-03-27T17:56:42.425 INFO:root:teuthology version: 1.0.0-b3a6b1a
2020-03-27T17:56:42.426 DEBUG:teuthology.run:Teuthology command: teuthology /tmp/teuthology-worker.f_vArZ.tmp -- --verbose --lock --description rados:thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-install/luminous.yaml backoff/peering.yaml ceph.yaml clusters/{openstack.yaml three-plus-one.yaml} d-balancer/crush-compat.yaml distro$/{centos_7.6.yaml} msgr-failures/osd-delay.yaml rados.yaml thrashers/pggrow.yaml thrashosds-health.yaml workloads/radosbench.yaml} --name sage-2020-03-27_17:51:50-rados:thrash-old-clients-revert-dec-distro-basic-smithi --block --owner scheduled_sage@teuthology --archive /home/teuthworker/archive/sage-2020-03-27_17:51:50-rados:thrash-old-clients-revert-dec-distro-basic-smithi/4895622
2020-03-27T17:56:42.469 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-03-27T17:56:42.529 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/sage-2020-03-27_17:51:50-rados:thrash-old-clients-revert-dec-distro-basic-smithi/4895622
  branch: revert-dec
  description: rados:thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size.yaml
    1-install/luminous.yaml backoff/peering.yaml ceph.yaml clusters/{openstack.yaml
    three-plus-one.yaml} d-balancer/crush-compat.yaml distro$/{centos_7.6.yaml} msgr-failures/osd-delay.yaml
    rados.yaml thrashers/pggrow.yaml thrashosds-health.yaml workloads/radosbench.yaml}
  email: null
  first_in_suite: false
  job_id: '4895622'
  kernel:
    kdb: true
    sha1: distro
  last_in_suite: false
  machine_type: smithi
  name: sage-2020-03-27_17:51:50-rados:thrash-old-clients-revert-dec-distro-basic-smithi
  nuke-on-error: true
  openstack:
  - volumes:
      count: 4
      size: 30
  - volumes:
      count: 4
      size: 10
  os_type: centos
  os_version: '7.6'
  overrides:
    admin_socket:
      branch: revert-dec
    ceph:
      conf:
        client.2:
          debug ms: 1
          debug objecter: 20
          debug rados: 20
        global:
          mon client directed command retry: 5
          mon warn on msgr2 not enabled: false
          ms inject delay max: 1
          ms inject delay probability: 0.005
          ms inject delay type: osd
          ms inject internal delays: 0.002
          ms inject socket failures: 2500
          osd_pool_default_min_size: 2
          osd_pool_default_size: 2
        mgr:
          debug mgr: 20
          debug ms: 1
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
          mon min osdmap epochs: 50
          mon osdmap full prune interval: 2
          mon osdmap full prune min: 15
          mon osdmap full prune txsize: 2
          paxos service trim min: 10
        osd:
          debug filestore: 20
          debug journal: 20
          debug ms: 20
          debug osd: 25
          filestore odsync write: true
          osd backoff on peering: true
          osd debug verify cached snaps: true
          osd debug verify missing on start: true
          osd max backfills: 2
          osd max markdown count: 1000
          osd op queue: debug_random
          osd op queue cut off: debug_random
          osd scrub max interval: 120
          osd scrub min interval: 60
          osd shutdown pgref assert: true
          osd snap trim sleep: 0.5
      log-whitelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      - \(REQUEST_STUCK\)
      - \(MON_DOWN\)
      - \(OSD_SLOW_PING_TIME
      - but it is still running
      - objects unfound and apparently lost
      - overall HEALTH_
      - \(OSDMAP_FLAGS\)
      - \(OSD_
      - \(PG_
      - \(POOL_
      - \(CACHE_POOL_
      - \(SMALLER_PGP_NUM\)
      - \(OBJECT_
      - \(SLOW_OPS\)
      - \(REQUEST_SLOW\)
      - \(TOO_FEW_PGS\)
      - slow request
      mon_bind_msgr2: false
      sha1: df873c1654814c23dcb68f897a012936832bfab9
    ceph-deploy:
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon:
          osd default pool size: 2
    install:
      ceph:
        sha1: df873c1654814c23dcb68f897a012936832bfab9
    selinux:
      whitelist:
      - scontext=system_u:system_r:logrotate_t:s0
    workunit:
      branch: revert-dec
      sha1: df873c1654814c23dcb68f897a012936832bfab9
  owner: scheduled_sage@teuthology
  priority: 50
  repo: git://git.ceph.com/ceph-ci.git
  roles:
  - - mon.a
    - mgr.y
    - osd.0
    - osd.1
    - osd.2
    - osd.3
    - client.0
  - - mon.b
    - mgr.x
    - osd.4
    - osd.5
    - osd.6
    - osd.7
    - client.1
  - - mon.c
    - osd.8
    - osd.9
    - osd.10
    - osd.11
    - client.2
  sha1: df873c1654814c23dcb68f897a012936832bfab9
  suite: rados:thrash-old-clients
  suite_branch: revert-dec
  suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_revert-dec/qa
  suite_relpath: qa
  suite_repo: git://git.ceph.com/ceph-ci.git
  suite_sha1: df873c1654814c23dcb68f897a012936832bfab9
  tasks:
  - install:
      branch: luminous
      exclude_packages:
      - librados3
      - ceph-mgr-dashboard
      - ceph-mgr-diskprediction-local
      - ceph-mgr-rook
      - ceph-mgr-cephadm
      - cephadm
      - ceph-immutable-object-cache
      - ceph-base
      - python3-rados
      - python3-rgw
      - python3-rbd
      - python3-cephfs
      - librados-devel
      extra_packages:
      - librados2
      - python-rados
      - python-rgw
      - python-rbd
      - python-cephfs
  - cephadm: null
  - exec:
      mon.a:
      - while ! ceph balancer status ; do sleep 1 ; done
      - ceph balancer mode crush-compat
      - ceph balancer on
  - thrashosds:
      chance_pgnum_grow: 2
      chance_pgpnum_fix: 1
      timeout: 1200
  - full_sequential:
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
  teuthology_branch: master
  tube: smithi
  verbose: false
  worker_log: /home/teuthworker/archive/worker_logs/worker.smithi.2409
2020-03-27T17:56:42.530 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_revert-dec/qa; will attempt to use it
2020-03-27T17:56:42.530 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_revert-dec/qa/tasks
2020-03-27T17:56:42.530 INFO:teuthology.run_tasks:Running task internal.check_packages...
2020-03-27T17:56:42.532 INFO:teuthology.task.internal:Checking packages...
2020-03-27T17:56:42.545 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'basic' and ceph hash 'df873c1654814c23dcb68f897a012936832bfab9'
2020-03-27T17:56:42.545 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2020-03-27T17:56:42.545 INFO:teuthology.packaging:ref: None
2020-03-27T17:56:42.545 INFO:teuthology.packaging:tag: None
2020-03-27T17:56:42.545 INFO:teuthology.packaging:branch: revert-dec
2020-03-27T17:56:42.545 INFO:teuthology.packaging:sha1: df873c1654814c23dcb68f897a012936832bfab9
2020-03-27T17:56:42.546 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F7%2Fx86_64&ref=revert-dec
2020-03-27T17:56:42.782 INFO:teuthology.task.internal:Found packages for ceph version 15.2.0-377.gdf873c1
2020-03-27T17:56:42.782 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2020-03-27T17:56:42.790 INFO:teuthology.task.internal:no buildpackages task found
2020-03-27T17:56:42.790 INFO:teuthology.run_tasks:Running task internal.lock_machines...
2020-03-27T17:56:42.801 INFO:teuthology.task.internal.lock_machines:Locking machines...
2020-03-27T17:56:42.801 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-03-27T17:56:42.847 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T17:56:52.863 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T17:57:02.878 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T17:57:12.895 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T17:57:22.913 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T17:57:32.931 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T17:57:42.951 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T17:57:52.973 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T17:58:02.989 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T17:58:13.005 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T17:58:23.032 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T17:58:33.050 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T17:58:43.067 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 0)...
2020-03-27T17:58:53.085 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 0)...
2020-03-27T17:59:03.102 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 0)...
2020-03-27T17:59:13.119 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 0)...
2020-03-27T17:59:23.132 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 0)...
2020-03-27T17:59:33.149 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 0)...
2020-03-27T17:59:43.158 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 0)...
2020-03-27T17:59:53.172 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 0)...
2020-03-27T18:00:03.189 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 0)...
2020-03-27T18:00:13.233 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:00:24.336 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:00:34.356 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:00:44.373 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:00:54.391 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:01:04.410 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:01:14.428 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:01:24.446 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:01:34.469 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:01:44.486 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:01:54.501 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:02:04.518 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:02:14.528 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:02:24.544 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:02:34.561 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:02:44.577 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:02:54.595 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:03:04.607 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:03:14.623 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:03:24.642 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:03:34.661 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:03:44.679 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:03:54.693 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:04:04.713 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:04:14.737 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:04:24.764 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:04:34.781 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:04:44.802 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:04:54.817 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:05:04.836 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:05:14.865 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:05:24.883 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:05:34.902 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:05:44.918 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:05:54.936 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:06:04.955 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:06:14.971 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:06:24.993 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:06:35.009 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:06:45.031 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:06:55.041 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:07:05.062 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:07:15.085 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T18:07:25.107 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:07:35.121 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:07:45.138 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:07:55.153 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:08:05.170 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:08:15.187 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:08:25.200 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:08:35.217 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:08:45.233 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:08:55.251 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:09:05.265 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:09:15.285 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:09:25.296 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:09:35.313 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:09:45.330 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:09:55.346 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:10:05.357 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 0)...
2020-03-27T18:10:15.374 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:10:25.390 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:10:35.405 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:10:45.420 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:10:55.436 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:11:05.453 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:11:15.468 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:11:25.485 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:11:35.502 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:11:45.517 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:11:55.535 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:12:05.552 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:12:15.569 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:12:25.585 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:12:35.597 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:12:45.615 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:12:55.631 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:13:05.654 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:13:15.669 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:13:25.698 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:13:35.713 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:13:45.728 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:13:55.744 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:14:05.761 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T18:14:15.777 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:14:25.800 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:14:35.816 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:14:45.833 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:14:55.852 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:15:05.872 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:15:15.889 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:15:25.906 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:15:35.921 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:15:45.938 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:15:55.953 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:16:05.970 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T18:16:15.995 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:16:26.009 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:16:36.026 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:16:46.040 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:16:56.059 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:17:06.079 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:17:16.114 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:17:26.142 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:17:36.162 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:17:46.177 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:17:56.194 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:18:06.217 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:18:16.243 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:18:26.260 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:18:36.278 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:18:46.296 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:18:56.313 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:19:06.331 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:19:16.349 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:19:26.366 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:19:36.381 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:19:46.397 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:19:56.413 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:20:06.430 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:20:16.446 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:20:26.479 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:20:36.498 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:20:46.508 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:20:56.526 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:21:06.557 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:21:16.574 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:21:26.598 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:21:36.621 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T18:21:46.639 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:21:56.657 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:22:06.675 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:22:16.697 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:22:26.713 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:22:36.731 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:22:46.747 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:22:56.763 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:23:06.779 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:23:16.795 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:23:26.812 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:23:36.828 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:23:46.853 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:23:56.869 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:24:06.886 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:24:16.911 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:24:26.929 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:24:36.952 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:24:46.969 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:24:56.987 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:25:07.008 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:25:17.025 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:25:27.041 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:25:37.057 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:25:47.074 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:25:57.090 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:26:07.104 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:26:17.122 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:26:27.140 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:26:37.165 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:26:47.189 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:26:57.205 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:27:07.223 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:27:17.245 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:27:27.262 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:27:37.278 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:27:47.295 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:27:57.314 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:28:07.328 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:28:17.342 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:28:27.353 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:28:37.377 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:28:47.500 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:28:57.518 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:29:07.534 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:29:17.555 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:29:27.575 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:29:37.589 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T18:29:47.607 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:29:57.623 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:30:07.641 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:30:17.660 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:30:27.677 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:30:37.702 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:30:47.718 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:30:57.733 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:31:07.755 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:31:17.773 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:31:27.790 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:31:37.808 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:31:47.828 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:31:57.860 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:32:07.877 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:32:17.894 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:32:27.912 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:32:37.930 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:32:47.945 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:32:57.966 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:33:07.982 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:33:18.003 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:33:28.022 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:33:38.039 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:33:48.056 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:33:58.074 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:34:08.091 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 1)...
2020-03-27T18:34:18.106 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:34:28.128 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:34:38.145 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:34:48.164 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:34:58.185 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:35:08.202 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:35:18.224 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:35:28.242 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:35:38.256 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:35:48.272 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:35:58.289 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:36:08.313 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:36:18.327 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:36:28.340 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:36:38.356 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:36:48.372 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:36:58.390 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:37:08.405 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:37:18.420 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:37:28.439 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:37:38.453 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:37:48.469 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:37:58.487 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:38:08.504 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:38:18.524 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:38:28.540 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:38:38.558 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:38:48.576 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:38:58.592 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:39:08.619 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:39:18.634 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...