2020-03-27T18:07:33.001 INFO:root:teuthology version: 1.0.0-b3a6b1a
2020-03-27T18:07:33.002 DEBUG:teuthology.run:Teuthology command: teuthology /tmp/teuthology-worker.3KkSdG.tmp -- --verbose --lock --description rados:thrash-old-clients/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-install/mimic-v1only.yaml backoff/normal.yaml ceph.yaml clusters/{openstack.yaml three-plus-one.yaml} d-balancer/off.yaml distro$/{centos_7.6.yaml} msgr-failures/few.yaml rados.yaml thrashers/pggrow.yaml thrashosds-health.yaml workloads/radosbench.yaml} --name sage-2020-03-27_17:51:50-rados:thrash-old-clients-revert-dec-distro-basic-smithi --block --owner scheduled_sage@teuthology --archive /home/teuthworker/archive/sage-2020-03-27_17:51:50-rados:thrash-old-clients-revert-dec-distro-basic-smithi/4895633
2020-03-27T18:07:33.047 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-03-27T18:07:33.118 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/sage-2020-03-27_17:51:50-rados:thrash-old-clients-revert-dec-distro-basic-smithi/4895633
  branch: revert-dec
  description: rados:thrash-old-clients/{0-size-min-size-overrides/3-size-2-min-size.yaml
    1-install/mimic-v1only.yaml backoff/normal.yaml ceph.yaml clusters/{openstack.yaml
    three-plus-one.yaml} d-balancer/off.yaml distro$/{centos_7.6.yaml} msgr-failures/few.yaml
    rados.yaml thrashers/pggrow.yaml thrashosds-health.yaml workloads/radosbench.yaml}
  email: null
  first_in_suite: false
  job_id: '4895633'
  kernel:
    kdb: true
    sha1: distro
  last_in_suite: false
  machine_type: smithi
  name: sage-2020-03-27_17:51:50-rados:thrash-old-clients-revert-dec-distro-basic-smithi
  nuke-on-error: true
  openstack:
  - volumes:
      count: 4
      size: 30
  - volumes:
      count: 4
      size: 10
  os_type: centos
  os_version: '7.6'
  overrides:
    admin_socket:
      branch: revert-dec
    ceph:
      conf:
        client.2:
          debug ms: 1
          debug objecter: 20
          debug rados: 20
        global:
          mon client directed command retry: 5
          ms bind msgr2: false
          ms inject socket failures: 5000
          ms type: async
          osd_pool_default_min_size: 2
          osd_pool_default_size: 3
        mgr:
          debug mgr: 20
          debug ms: 1
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
          mon min osdmap epochs: 50
          mon osdmap full prune interval: 2
          mon osdmap full prune min: 15
          mon osdmap full prune txsize: 2
          paxos service trim min: 10
        osd:
          debug filestore: 20
          debug journal: 20
          debug ms: 20
          debug osd: 25
          filestore odsync write: true
          osd debug verify cached snaps: true
          osd debug verify missing on start: true
          osd heartbeat use min delay socket: true
          osd max backfills: 2
          osd max markdown count: 1000
          osd op queue: debug_random
          osd op queue cut off: debug_random
          osd scrub max interval: 120
          osd scrub min interval: 60
          osd shutdown pgref assert: true
          osd snap trim sleep: 0.5
      log-whitelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      - \(MON_DOWN\)
      - \(OSD_SLOW_PING_TIME
      - but it is still running
      - objects unfound and apparently lost
      - overall HEALTH_
      - \(OSDMAP_FLAGS\)
      - \(OSD_
      - \(PG_
      - \(POOL_
      - \(CACHE_POOL_
      - \(SMALLER_PGP_NUM\)
      - \(OBJECT_
      - \(SLOW_OPS\)
      - \(REQUEST_SLOW\)
      - \(TOO_FEW_PGS\)
      - slow request
      mon_bind_msgr2: false
      sha1: df873c1654814c23dcb68f897a012936832bfab9
    ceph-deploy:
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon:
          osd default pool size: 2
    install:
      ceph:
        sha1: df873c1654814c23dcb68f897a012936832bfab9
    selinux:
      whitelist:
      - scontext=system_u:system_r:logrotate_t:s0
    thrashosds:
      min_in: 4
    workunit:
      branch: revert-dec
      sha1: df873c1654814c23dcb68f897a012936832bfab9
  owner: scheduled_sage@teuthology
  priority: 50
  repo: git://git.ceph.com/ceph-ci.git
  roles:
  - - mon.a
    - mgr.y
    - osd.0
    - osd.1
    - osd.2
    - osd.3
    - client.0
  - - mon.b
    - mgr.x
    - osd.4
    - osd.5
    - osd.6
    - osd.7
    - client.1
  - - mon.c
    - osd.8
    - osd.9
    - osd.10
    - osd.11
    - client.2
  sha1: df873c1654814c23dcb68f897a012936832bfab9
  suite: rados:thrash-old-clients
  suite_branch: revert-dec
  suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_revert-dec/qa
  suite_relpath: qa
  suite_repo: git://git.ceph.com/ceph-ci.git
  suite_sha1: df873c1654814c23dcb68f897a012936832bfab9
  tasks:
  - install:
      branch: mimic
      exclude_packages:
      - librados3
      - ceph-mgr-dashboard
      - ceph-mgr-diskprediction-local
      - ceph-mgr-rook
      - ceph-mgr-cephadm
      - cephadm
      - ceph-immutable-object-cache
      - ceph-base
      - python3-rados
      - python3-rgw
      - python3-rbd
      - python3-cephfs
      - librados-devel
      extra_packages:
      - librados2
      - python-rados
      - python-rgw
      - python-rbd
      - python-cephfs
  - cephadm: null
  - thrashosds:
      chance_pgnum_grow: 2
      chance_pgpnum_fix: 1
      timeout: 1200
  - full_sequential:
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
    - radosbench:
        clients:
        - client.2
        time: 90
  teuthology_branch: master
  tube: smithi
  verbose: false
  worker_log: /home/teuthworker/archive/worker_logs/worker.smithi.2754
2020-03-27T18:07:33.118 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_revert-dec/qa; will attempt to use it
2020-03-27T18:07:33.120 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_revert-dec/qa/tasks
2020-03-27T18:07:33.120 INFO:teuthology.run_tasks:Running task internal.check_packages...
2020-03-27T18:07:33.121 INFO:teuthology.task.internal:Checking packages...
2020-03-27T18:07:33.135 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'basic' and ceph hash 'df873c1654814c23dcb68f897a012936832bfab9'
2020-03-27T18:07:33.136 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2020-03-27T18:07:33.136 INFO:teuthology.packaging:ref: None
2020-03-27T18:07:33.136 INFO:teuthology.packaging:tag: None
2020-03-27T18:07:33.136 INFO:teuthology.packaging:branch: revert-dec
2020-03-27T18:07:33.136 INFO:teuthology.packaging:sha1: df873c1654814c23dcb68f897a012936832bfab9
2020-03-27T18:07:33.136 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F7%2Fx86_64&ref=revert-dec
2020-03-27T18:07:33.387 INFO:teuthology.task.internal:Found packages for ceph version 15.2.0-377.gdf873c1
2020-03-27T18:07:33.557 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2020-03-27T18:07:33.567 INFO:teuthology.task.internal:no buildpackages task found
2020-03-27T18:07:33.567 INFO:teuthology.run_tasks:Running task internal.lock_machines...
2020-03-27T18:07:33.705 INFO:teuthology.task.internal.lock_machines:Locking machines...
2020-03-27T18:07:33.705 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-03-27T18:07:33.766 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:07:43.782 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:07:53.797 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:08:03.808 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:08:13.827 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:08:23.842 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:08:33.858 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:08:43.876 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:08:53.889 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:09:03.906 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:09:13.918 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:09:23.933 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:09:33.950 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:09:43.964 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:09:53.981 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:10:03.998 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 0)...
2020-03-27T18:10:14.015 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:10:24.033 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:10:34.047 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:10:44.063 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:10:54.077 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:11:04.093 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:11:14.109 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:11:24.126 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:11:34.142 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:11:44.163 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:11:54.177 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:12:04.197 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:12:14.214 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:12:24.229 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:12:34.245 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:12:44.266 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:12:54.286 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:13:04.306 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:13:14.320 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:13:24.337 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:13:34.355 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:13:44.376 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:13:54.392 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:14:04.410 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 4)...
2020-03-27T18:14:14.426 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:14:24.443 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:14:34.469 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:14:44.485 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:14:54.513 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:15:04.525 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:15:14.538 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:15:24.553 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:15:34.569 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:15:44.585 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:15:54.603 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:16:04.617 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:16:14.635 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:16:24.656 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:16:34.677 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:16:44.696 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:16:54.716 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:17:04.739 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:17:14.757 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:17:24.776 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:17:34.793 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:17:44.810 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:17:54.825 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:18:04.842 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:18:14.860 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:18:24.877 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:18:34.896 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:18:44.913 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:18:54.930 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:19:04.964 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:19:14.983 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:19:25.002 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:19:35.021 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:19:45.038 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:19:55.056 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:20:05.075 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:20:15.092 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:20:25.116 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:20:35.136 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:20:45.154 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:20:55.171 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:21:05.191 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:21:15.209 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:21:25.226 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:21:35.242 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:21:45.257 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:21:55.277 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:22:05.295 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:22:15.315 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:22:25.329 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:22:35.355 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:22:45.370 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:22:55.385 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:23:05.402 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:23:15.418 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:23:25.434 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:23:35.449 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:23:45.466 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:23:55.481 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:24:05.503 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:24:15.521 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:24:25.540 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:24:35.557 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:24:45.575 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:24:55.599 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:25:05.624 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:25:15.641 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:25:25.657 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:25:35.673 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:25:45.689 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:25:55.705 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:26:05.724 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:26:15.741 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:26:25.759 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:26:35.775 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:26:45.791 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:26:55.809 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:27:05.825 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:27:15.854 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:27:25.871 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:27:35.888 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:27:45.906 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:27:55.922 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:28:05.939 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:28:15.962 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:28:25.978 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:28:35.990 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:28:46.015 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:28:56.034 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:29:06.049 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:29:16.066 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:29:26.083 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:29:36.098 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:29:46.112 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:29:56.132 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:30:06.149 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:30:16.167 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:30:26.180 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:30:36.197 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:30:46.212 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:30:56.234 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:31:06.249 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:31:16.265 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:31:26.281 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:31:36.296 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:31:46.315 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:31:56.329 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:32:06.344 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:32:16.371 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:32:26.406 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:32:36.421 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 1)...
2020-03-27T18:32:46.437 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:32:56.461 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:33:06.500 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:33:16.516 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:33:26.534 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:33:36.552 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:33:46.569 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:33:56.585 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 2)...
2020-03-27T18:34:06.601 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 1)...
2020-03-27T18:34:16.617 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:34:26.633 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:34:36.655 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:34:46.672 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:34:56.693 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:35:06.730 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:35:16.746 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:35:26.766 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:35:36.781 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:35:46.803 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 3, have 3)...
2020-03-27T18:35:56.818 DEBUG:teuthology.lock.ops:lock_many request: {'count': 3, 'locked_by': 'scheduled_sage@teuthology', 'description': '/home/teuthworker/archive/sage-2020-03-27_17:51:50-rados:thrash-old-clients-revert-dec-distro-basic-smithi/4895633', 'machine_type': 'smithi'}
2020-03-27T18:35:56.845 DEBUG:teuthology.lock.ops:locked ubuntu@smithi047.front.sepia.ceph.com, ubuntu@smithi042.front.sepia.ceph.com, ubuntu@smithi071.front.sepia.ceph.com
2020-03-27T18:35:56.988 DEBUG:teuthology.exit:Installing handler: Handler(exiter=<teuthology.exit.Exiter object at 0x7f7cdf5f6fd0>, func=<function kill_console_loggers at 0x7f7cdf68ccf8>, signals=[15])
2020-03-27T18:35:57.195 INFO:teuthology.provision.fog.smithi071:Scheduling deploy of centos 7.6
2020-03-27T18:35:57.202 INFO:teuthology.provision.fog.smithi042:Scheduling deploy of centos 7.6
2020-03-27T18:35:57.205 INFO:teuthology.provision.fog.smithi047:Scheduling deploy of centos 7.6
2020-03-27T18:35:57.501 INFO:teuthology.orchestra.console:Power off smithi042
2020-03-27T18:35:57.502 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi042.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2020-03-27T18:35:57.525 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi042.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T18:35:57.576 INFO:teuthology.orchestra.console:Power off smithi047
2020-03-27T18:35:57.577 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi047.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2020-03-27T18:35:57.593 INFO:teuthology.orchestra.console:Power off smithi071
2020-03-27T18:35:57.593 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi071.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2020-03-27T18:35:57.602 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi047.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T18:35:57.613 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi071.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T18:35:57.701 INFO:teuthology.orchestra.console:Power off for smithi042 completed
2020-03-27T18:35:57.723 INFO:teuthology.orchestra.console:Power off for smithi047 completed
2020-03-27T18:35:57.733 INFO:teuthology.orchestra.console:Power off for smithi071 completed
2020-03-27T18:35:57.803 INFO:teuthology.orchestra.console:Power on smithi042
2020-03-27T18:35:57.803 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi042.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power on
2020-03-27T18:35:57.823 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi042.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T18:35:57.830 INFO:teuthology.orchestra.console:Power on smithi047
2020-03-27T18:35:57.830 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi047.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power on
2020-03-27T18:35:57.837 INFO:teuthology.orchestra.console:Power on smithi071
2020-03-27T18:35:57.837 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi071.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power on
2020-03-27T18:35:57.850 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi047.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T18:35:57.861 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi071.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T18:36:01.873 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi042.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T18:36:01.902 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi047.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T18:36:01.918 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi071.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T18:36:02.128 INFO:teuthology.orchestra.console:Power on for smithi071 completed
2020-03-27T18:36:02.229 INFO:teuthology.provision.fog.smithi071:Waiting for deploy to finish
2020-03-27T18:36:05.985 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi042.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T18:36:06.021 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi047.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T18:36:06.195 INFO:teuthology.orchestra.console:Power on for smithi042 completed
2020-03-27T18:36:06.239 INFO:teuthology.orchestra.console:Power on for smithi047 completed
2020-03-27T18:36:06.313 INFO:teuthology.provision.fog.smithi042:Waiting for deploy to finish
2020-03-27T18:36:06.341 INFO:teuthology.provision.fog.smithi047:Waiting for deploy to finish
2020-03-27T18:38:37.227 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi042.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T18:38:37.230 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi047.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T18:38:46.289 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi042.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T18:38:46.309 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi047.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T18:38:55.373 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi042.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T18:38:55.374 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi047.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T18:39:03.293 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi071.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T18:39:04.458 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi042.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T18:39:04.459 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi047.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T18:39:12.361 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi071.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T18:39:13.510 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi042.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T18:39:13.511 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi047.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T18:39:19.913 DEBUG:teuthology.exit:Got signal 15; running 1 handler...
2020-03-27T18:39:19.914 DEBUG:teuthology.task.console_log:Killing console logger for smithi042
2020-03-27T18:39:19.914 DEBUG:teuthology.task.console_log:Killing console logger for smithi047
2020-03-27T18:39:19.914 DEBUG:teuthology.task.console_log:Killing console logger for smithi071
2020-03-27T18:39:19.915 DEBUG:teuthology.exit:Finished running handlers