2020-03-27T13:50:05.690 INFO:root:teuthology version: 1.0.0-b3a6b1a
2020-03-27T13:50:05.691 DEBUG:teuthology.run:Teuthology command: teuthology /tmp/teuthology-worker.i6xfdc.tmp -- --verbose --lock --description rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/default/{default.yaml thrashosds-health.yaml} msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml tasks/rados_cls_all.yaml validater/valgrind.yaml} --name kchai-2020-03-27_13:29:55-rados-nautilus-kefu2-testing-2020-03-27-1709-distro-basic-smithi --block --owner scheduled_kchai@teuthology --archive /home/teuthworker/archive/kchai-2020-03-27_13:29:55-rados-nautilus-kefu2-testing-2020-03-27-1709-distro-basic-smithi/4894915
2020-03-27T13:50:05.732 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-03-27T13:50:05.819 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/kchai-2020-03-27_13:29:55-rados-nautilus-kefu2-testing-2020-03-27-1709-distro-basic-smithi/4894915
  branch: nautilus-kefu2-testing-2020-03-27-1709
  description: rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/default/{default.yaml
    thrashosds-health.yaml} msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml
    rados.yaml tasks/rados_cls_all.yaml validater/valgrind.yaml}
  email: tchaikov@gmail.com
  first_in_suite: false
  job_id: '4894915'
  kernel:
    kdb: true
    sha1: distro
  last_in_suite: false
  machine_type: smithi
  name: kchai-2020-03-27_13:29:55-rados-nautilus-kefu2-testing-2020-03-27-1709-distro-basic-smithi
  nuke-on-error: true
  openstack:
  - volumes:
      count: 4
      size: 10
  - volumes:
      count: 4
      size: 10
  os_type: centos
  overrides:
    admin_socket:
      branch: nautilus-kefu2-testing-2020-03-27-1709
    ceph:
      conf:
        global:
          debug refs: 5
          ms inject socket failures: 5000
          ms type: async
          osd heartbeat grace: 40
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
          mon min osdmap epochs: 50
          mon osd crush smoke test: false
          mon osdmap full prune interval: 2
          mon osdmap full prune min: 15
          mon osdmap full prune txsize: 2
          paxos service trim min: 10
        osd:
          bdev async discard: true
          bdev enable discard: true
          bluestore allocator: bitmap
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          debug bluefs: 20
          debug bluestore: 20
          debug filestore: 20
          debug journal: 20
          debug ms: 1
          debug osd: 25
          debug rocksdb: 10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd debug verify cached snaps: true
          osd debug verify missing on start: true
          osd failsafe full ratio: 0.95
          osd fast shutdown: false
          osd objectstore: bluestore
          osd op queue: debug_random
          osd op queue cut off: debug_random
          osd shutdown pgref assert: true
          osd_class_default_list: cephfs hello journal lock log numops rbd refcount
            rgw sdk timeindex user version
          osd_class_load_list: cephfs hello journal lock log numops rbd refcount rgw
            sdk timeindex user version
      fs: xfs
      log-whitelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      - but it is still running
      - objects unfound and apparently lost
      - overall HEALTH_
      - \(OSDMAP_FLAGS\)
      - \(OSD_
      - \(PG_
      - \(POOL_
      - \(CACHE_POOL_
      - \(SMALLER_PGP_NUM\)
      - \(OBJECT_
      - \(SLOW_OPS\)
      - \(REQUEST_SLOW\)
      - \(TOO_FEW_PGS\)
      - slow request
      - \(OSD_SLOW_PING_TIME
      - overall HEALTH_
      - \(PG_
      sha1: e9aaac6a13136ef54c1c1edf22f45ec35b28b19a
      valgrind:
        mds:
        - --tool=memcheck
        mon:
        - --tool=memcheck
        - --leak-check=full
        - --show-reachable=yes
        osd:
        - --tool=memcheck
    ceph-deploy:
      bluestore: true
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon:
          osd default pool size: 2
        osd:
          bdev async discard: true
          bdev enable discard: true
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          debug bluefs: 20
          debug bluestore: 20
          debug rocksdb: 10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd failsafe full ratio: 0.95
          osd objectstore: bluestore
      fs: xfs
    install:
      ceph:
        debuginfo: true
        flavor: notcmalloc
        sha1: e9aaac6a13136ef54c1c1edf22f45ec35b28b19a
    thrashosds:
      bdev_inject_crash: 2
      bdev_inject_crash_probability: 0.5
    workunit:
      branch: nautilus-kefu2-testing-2020-03-27-1709
      sha1: e9aaac6a13136ef54c1c1edf22f45ec35b28b19a
  owner: scheduled_kchai@teuthology
  priority: 90
  repo: git://git.ceph.com/ceph-ci.git
  roles:
  - - mon.a
    - mon.c
    - mgr.y
    - osd.0
    - osd.1
    - osd.2
    - osd.3
    - client.0
  - - mon.b
    - mgr.x
    - osd.4
    - osd.5
    - osd.6
    - osd.7
    - client.1
  sha1: e9aaac6a13136ef54c1c1edf22f45ec35b28b19a
  suite: rados
  suite_branch: nautilus-kefu2-testing-2020-03-27-1709
  suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_nautilus-kefu2-testing-2020-03-27-1709/qa
  suite_relpath: qa
  suite_repo: git://git.ceph.com/ceph-ci.git
  suite_sha1: e9aaac6a13136ef54c1c1edf22f45ec35b28b19a
  tasks:
  - install: null
  - ceph: null
  - thrashosds:
      chance_pgnum_grow: 1
      chance_pgnum_shrink: 1
      chance_pgpnum_fix: 1
      timeout: 1200
  - workunit:
      clients:
        client.0:
        - cls
  teuthology_branch: master
  tube: smithi
  verbose: false
  worker_log: /home/teuthworker/archive/worker_logs/worker.smithi.27840
2020-03-27T13:50:05.819 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_nautilus-kefu2-testing-2020-03-27-1709/qa; will attempt to use it
2020-03-27T13:50:05.820 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_nautilus-kefu2-testing-2020-03-27-1709/qa/tasks
2020-03-27T13:50:05.820 INFO:teuthology.run_tasks:Running task internal.check_packages...
2020-03-27T13:50:05.822 INFO:teuthology.task.internal:Checking packages...
2020-03-27T13:50:05.835 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'notcmalloc' and ceph hash 'e9aaac6a13136ef54c1c1edf22f45ec35b28b19a'
2020-03-27T13:50:05.836 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2020-03-27T13:50:05.836 INFO:teuthology.packaging:ref: None
2020-03-27T13:50:05.836 INFO:teuthology.packaging:tag: None
2020-03-27T13:50:05.836 INFO:teuthology.packaging:branch: nautilus-kefu2-testing-2020-03-27-1709
2020-03-27T13:50:05.836 INFO:teuthology.packaging:sha1: e9aaac6a13136ef54c1c1edf22f45ec35b28b19a
2020-03-27T13:50:05.836 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=notcmalloc&distros=centos%2F7%2Fx86_64&ref=nautilus-kefu2-testing-2020-03-27-1709
2020-03-27T13:50:06.070 INFO:teuthology.task.internal:Found packages for ceph version 14.2.8-299.ge9aaac6
2020-03-27T13:50:06.070 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2020-03-27T13:50:06.078 INFO:teuthology.task.internal:no buildpackages task found
2020-03-27T13:50:06.079 INFO:teuthology.run_tasks:Running task internal.lock_machines...
2020-03-27T13:50:06.287 INFO:teuthology.task.internal.lock_machines:Locking machines...
2020-03-27T13:50:06.287 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-03-27T13:50:06.349 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 2, have 3)...
2020-03-27T13:50:16.365 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 2, have 3)...
2020-03-27T13:50:26.383 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 2 + 2, have 3)...
2020-03-27T13:50:36.400 DEBUG:teuthology.lock.ops:lock_many request: {'count': 2, 'locked_by': 'scheduled_kchai@teuthology', 'description': '/home/teuthworker/archive/kchai-2020-03-27_13:29:55-rados-nautilus-kefu2-testing-2020-03-27-1709-distro-basic-smithi/4894915', 'machine_type': 'smithi'}
2020-03-27T13:50:36.417 DEBUG:teuthology.lock.ops:locked ubuntu@smithi075.front.sepia.ceph.com, ubuntu@smithi186.front.sepia.ceph.com
2020-03-27T13:50:36.485 DEBUG:teuthology.exit:Installing handler: Handler(exiter=<teuthology.exit.Exiter object at 0x7f6d0954afd0>, func=<function kill_console_loggers at 0x7f6d0b6452a8>, signals=[15])
2020-03-27T13:50:37.063 INFO:teuthology.provision.fog.smithi186:Scheduling deploy of centos 7.6
2020-03-27T13:50:37.066 INFO:teuthology.provision.fog.smithi075:Scheduling deploy of centos 7.6
2020-03-27T13:50:37.405 INFO:teuthology.orchestra.console:Power off smithi186
2020-03-27T13:50:37.405 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi186.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2020-03-27T13:50:37.433 INFO:teuthology.orchestra.console:Power off smithi075
2020-03-27T13:50:37.434 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi075.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2020-03-27T13:50:37.442 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi186.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T13:50:37.455 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi075.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T13:50:37.564 INFO:teuthology.orchestra.console:Power off for smithi186 completed
2020-03-27T13:50:37.666 INFO:teuthology.orchestra.console:Power on smithi186
2020-03-27T13:50:37.666 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi186.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power on
2020-03-27T13:50:37.689 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi186.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T13:50:41.481 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi075.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T13:50:41.709 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi186.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T13:50:45.594 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi075.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T13:50:45.803 INFO:teuthology.orchestra.console:Power off for smithi075 completed
2020-03-27T13:50:45.819 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi186.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T13:50:45.904 INFO:teuthology.orchestra.console:Power on smithi075
2020-03-27T13:50:45.905 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi075.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power on
2020-03-27T13:50:45.925 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi075.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T13:50:46.036 INFO:teuthology.orchestra.console:Power on for smithi186 completed
2020-03-27T13:50:46.136 INFO:teuthology.provision.fog.smithi186:Waiting for deploy to finish
2020-03-27T13:50:49.949 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi075.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T13:50:54.065 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi075.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-03-27T13:50:54.274 INFO:teuthology.orchestra.console:Power on for smithi075 completed
2020-03-27T13:50:54.375 INFO:teuthology.provision.fog.smithi075:Waiting for deploy to finish
2020-03-27T13:53:32.285 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi186.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:53:40.489 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi075.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:53:41.350 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi186.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:53:49.545 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi075.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:53:50.406 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi186.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:53:58.601 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi075.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:53:59.466 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi186.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:54:07.657 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi075.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:54:08.518 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi186.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:54:16.713 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi075.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:54:17.574 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi186.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:54:25.769 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi075.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:54:26.630 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi186.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:54:34.822 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi075.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:54:35.686 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi186.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:54:43.882 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi075.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:54:52.937 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi075.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:55:01.993 DEBUG:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'smithi075.front.sepia.ceph.com', 'timeout': 60}
2020-03-27T13:55:34.619 DEBUG:teuthology.exit:Got signal 15; running 1 handler...
2020-03-27T13:55:34.619 DEBUG:teuthology.task.console_log:Killing console logger for smithi186
2020-03-27T13:55:34.620 DEBUG:teuthology.task.console_log:Killing console logger for smithi075
2020-03-27T13:55:34.620 DEBUG:teuthology.exit:Finished running handlers