2020-11-12T17:20:47.751 INFO:root:teuthology version: 1.0.0-357fd4a
2020-11-12T17:20:47.757 DEBUG:teuthology.run:Teuthology command: teuthology --name yuriw-2020-11-12_17:03:16-rados-octopus-distro-basic-smithi --description rados/thrash-erasure-code-overwrites/{bluestore-bitmap ceph clusters/{fixed-2 openstack} fast/fast msgr-failures/few rados recovery-overrides/{more-async-recovery} supported-random-distro$/{centos_latest} thrashers/default thrashosds-health workloads/ec-snaps-few-objects-overwrites} --lock --verbose --archive /home/teuthworker/archive/yuriw-2020-11-12_17:03:16-rados-octopus-distro-basic-smithi/5616044 --block --owner scheduled_yuriw@teuthology -- /tmp/teuthology-worker.1k8ll2vr.tmp
2020-11-12T17:20:47.802 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-11-12T17:20:48.051 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/yuriw-2020-11-12_17:03:16-rados-octopus-distro-basic-smithi/5616044
  branch: octopus
  description: rados/thrash-erasure-code-overwrites/{bluestore-bitmap ceph clusters/{fixed-2
    openstack} fast/fast msgr-failures/few rados recovery-overrides/{more-async-recovery}
    supported-random-distro$/{centos_latest} thrashers/default thrashosds-health workloads/ec-snaps-few-objects-overwrites}
  email: ceph-qa@ceph.io
  first_in_suite: false
  job_id: '5616044'
  kernel:
    kdb: true
    sha1: distro
  last_in_suite: false
  machine_type: smithi
  name: yuriw-2020-11-12_17:03:16-rados-octopus-distro-basic-smithi
  nuke-on-error: true
  openstack:
  - volumes:
      count: 4
      size: 10
  - volumes:
      count: 4
      size: 30
  os_type: centos
  os_version: '8.1'
  overrides:
    admin_socket:
      branch: octopus
    ceph:
      conf:
        global:
          enable experimental unrecoverable data corrupting features: '*'
          mon client directed command retry: 5
          ms inject socket failures: 5000
          osd pool default ec fast read: true
          osd_async_recovery_min_cost: 1
        mgr:
          debug mgr: 20
          debug ms: 1
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
        osd:
          bdev async discard: true
          bdev enable discard: true
          bluestore allocator: bitmap
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          debug bluefs: 20
          debug bluestore: 20
          debug filestore: 20
          debug journal: 20
          debug ms: 20
          debug osd: 25
          debug rocksdb: 10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd debug reject backfill probability: 0.1
          osd debug verify cached snaps: true
          osd debug verify missing on start: true
          osd failsafe full ratio: 0.95
          osd heartbeat use min delay socket: true
          osd max backfills: 2
          osd max markdown count: 1000
          osd objectstore: bluestore
          osd op queue: debug_random
          osd op queue cut off: debug_random
          osd scrub max interval: 120
          osd scrub min interval: 60
          osd shutdown pgref assert: true
      fs: xfs
      log-ignorelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      log-whitelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      - \(OSD_SLOW_PING_TIME
      - but it is still running
      - objects unfound and apparently lost
      - overall HEALTH_
      - \(OSDMAP_FLAGS\)
      - \(OSD_
      - \(PG_
      - \(POOL_
      - \(CACHE_POOL_
      - \(SMALLER_PGP_NUM\)
      - \(OBJECT_
      - \(SLOW_OPS\)
      - \(REQUEST_SLOW\)
      - \(TOO_FEW_PGS\)
      - slow request
      sha1: 31d4e76d0d584790763b4b1146b29ea4cfc2e9af
    ceph-deploy:
      bluestore: true
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon:
          osd default pool size: 2
        osd:
          bdev async discard: true
          bdev enable discard: true
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          debug bluefs: 20
          debug bluestore: 20
          debug rocksdb: 10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd failsafe full ratio: 0.95
          osd objectstore: bluestore
      fs: xfs
    install:
      ceph:
        sha1: 31d4e76d0d584790763b4b1146b29ea4cfc2e9af
    selinux:
      whitelist:
      - scontext=system_u:system_r:logrotate_t:s0
    thrashosds:
      bdev_inject_crash: 2
      bdev_inject_crash_probability: 0.5
      disable_objectstore_tool_tests: true
    workunit:
      branch: octopus
      sha1: 31d4e76d0d584790763b4b1146b29ea4cfc2e9af
  owner: scheduled_yuriw@teuthology
  priority: 99
  repo: https://github.com/ceph/ceph.git
  roles:
  - - mon.a
    - mon.c
    - mgr.y
    - osd.0
    - osd.1
    - osd.2
    - osd.3
    - client.0
    - node-exporter.a
  - - mon.b
    - mgr.x
    - osd.4
    - osd.5
    - osd.6
    - osd.7
    - client.1
    - prometheus.a
    - node-exporter.b
  sha1: 31d4e76d0d584790763b4b1146b29ea4cfc2e9af
  sleep_before_teardown: 0
  suite: rados
  suite_branch: octopus
  suite_path: /home/teuthworker/src/github.com_ceph_ceph_octopus/qa
  suite_relpath: qa
  suite_repo: https://github.com/ceph/ceph.git
  suite_sha1: 31d4e76d0d584790763b4b1146b29ea4cfc2e9af
  tasks:
  - install: null
  - ceph:
      conf:
        osd:
          debug monc: 20
  - thrashosds:
      chance_pgnum_grow: 1
      chance_pgnum_shrink: 1
      chance_pgpnum_fix: 1
      min_in: 4
      timeout: 1200
  - rados:
      clients:
      - client.0
      ec_pool: true
      erasure_code_use_overwrites: true
      objects: 50
      op_weights:
        copy_from: 50
        delete: 50
        read: 100
        rollback: 50
        snap_create: 50
        snap_remove: 50
        write: 100
      ops: 4000
  teuthology_branch: master
  timestamp: 2020-11-12_17:03:16
  tube: smithi
  user: yuriw
  verbose: true
  worker_log: /home/teuthworker/archive/worker_logs/worker.smithi.11578
2020-11-12T17:20:48.052 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_octopus/qa; will attempt to use it
2020-11-12T17:20:48.053 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_octopus/qa/tasks
2020-11-12T17:20:48.054 INFO:teuthology.run_tasks:Running task internal.check_packages...
2020-11-12T17:20:48.056 INFO:teuthology.task.internal:Checking packages...
2020-11-12T17:20:48.081 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'basic' and ceph hash '31d4e76d0d584790763b4b1146b29ea4cfc2e9af'
2020-11-12T17:20:48.081 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2020-11-12T17:20:48.082 INFO:teuthology.packaging:ref: None
2020-11-12T17:20:48.082 INFO:teuthology.packaging:tag: None
2020-11-12T17:20:48.082 INFO:teuthology.packaging:branch: octopus
2020-11-12T17:20:48.083 INFO:teuthology.packaging:sha1: 31d4e76d0d584790763b4b1146b29ea4cfc2e9af
2020-11-12T17:20:48.083 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=octopus
2020-11-12T17:20:48.424 INFO:teuthology.task.internal:Found packages for ceph version 15.2.5-585.g31d4e76d
2020-11-12T17:20:48.424 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2020-11-12T17:20:48.438 INFO:teuthology.task.internal:no buildpackages task found
2020-11-12T17:20:48.439 INFO:teuthology.run_tasks:Running task internal.lock_machines...
2020-11-12T17:20:48.451 INFO:teuthology.task.internal.lock_machines:Locking machines...
2020-11-12T17:20:48.452 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-11-12T17:20:48.506 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:20:58.523 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 1)...
2020-11-12T17:21:08.545 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 1)...
2020-11-12T17:21:18.578 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 1)...
2020-11-12T17:21:28.628 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:21:38.686 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:21:48.731 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:21:58.752 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:22:08.775 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:22:18.795 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:22:28.823 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:22:38.843 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:22:48.880 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:22:58.898 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:23:08.998 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:23:19.057 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:23:29.091 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:23:39.133 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:23:49.163 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:23:59.284 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:24:09.336 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:24:19.371 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:24:29.416 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:24:39.456 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:24:49.482 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:24:59.500 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:25:09.529 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-12T17:25:19.548 DEBUG:teuthology.lock.ops:lock_many request: {'locked_by': 'scheduled_yuriw@teuthology', 'count': 2, 'machine_type': 'smithi', 'description': '/home/teuthworker/archive/yuriw-2020-11-12_17:03:16-rados-octopus-distro-basic-smithi/5616044'}
2020-11-12T17:25:19.568 DEBUG:teuthology.lock.ops:locked ubuntu@smithi188.front.sepia.ceph.com, ubuntu@smithi038.front.sepia.ceph.com
2020-11-12T17:25:19.639 DEBUG:teuthology.exit:Installing handler: Handler(exiter=<teuthology.exit.Exiter object at 0x7f5ea6956da0>, func=<function ConsoleLog.start_logging.<locals>.kill_console_loggers at 0x7f5ea52deb70>, signals=[15])
2020-11-12T17:25:19.860 INFO:teuthology.provision.fog.smithi038:Scheduling deploy of centos 8.1
2020-11-12T17:25:19.864 INFO:teuthology.provision.fog.smithi188:Scheduling deploy of centos 8.1
2020-11-12T17:25:20.240 INFO:teuthology.orchestra.console:Power off smithi188
2020-11-12T17:25:20.241 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi188.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2020-11-12T17:25:20.259 INFO:teuthology.orchestra.console:Power off smithi038
2020-11-12T17:25:20.260 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi038.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2020-11-12T17:25:20.275 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi188.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-12T17:25:20.305 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi038.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-12T17:25:24.306 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi188.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-12T17:25:24.332 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi038.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-12T17:25:28.422 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi188.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-12T17:25:28.446 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi038.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-12T17:25:28.657 INFO:teuthology.orchestra.console:Power off for smithi038 completed
2020-11-12T17:25:28.758 INFO:teuthology.orchestra.console:Power on smithi038
2020-11-12T17:25:28.758 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi038.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power on
2020-11-12T17:25:28.790 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi038.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-12T17:25:32.534 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi188.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-12T17:25:32.746 INFO:teuthology.orchestra.console:Power off for smithi188 completed
2020-11-12T17:25:32.815 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi038.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-12T17:25:32.847 INFO:teuthology.orchestra.console:Power on smithi188
2020-11-12T17:25:32.848 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi188.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power on
2020-11-12T17:25:32.873 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi188.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-12T17:25:36.902 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi188.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-12T17:25:36.928 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi038.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-12T17:25:37.117 INFO:teuthology.orchestra.console:Power on for smithi188 completed
2020-11-12T17:25:37.147 INFO:teuthology.orchestra.console:Power on for smithi038 completed
2020-11-12T17:25:37.218 INFO:teuthology.provision.fog.smithi188:Waiting for deploy to finish
2020-11-12T17:25:37.248 INFO:teuthology.provision.fog.smithi038:Waiting for deploy to finish
2020-11-12T17:27:53.569 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi038.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:28:02.675 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi038.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:28:11.761 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi038.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:28:23.821 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi188.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:28:24.595 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi038.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:28:33.110 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi188.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:28:33.647 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi038.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:28:42.202 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi188.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:28:42.708 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi038.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:28:51.739 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi188.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:29:00.858 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi188.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:29:13.090 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi188.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:29:22.451 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi188.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:29:48.746 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi038.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:29:50.201 INFO:teuthology.orchestra.run:Running command with timeout 600
2020-11-12T17:29:50.329 INFO:teuthology.orchestra.run.smithi038:> while [ ! -e '/.cephlab_net_configured' ]; do sleep 5; done
2020-11-12T17:29:50.488 INFO:teuthology.provision.fog.smithi038:Node is ready
2020-11-12T17:29:50.495 INFO:teuthology.orchestra.run.smithi038:> hostname
2020-11-12T17:29:50.511 INFO:teuthology.orchestra.run.smithi038.stdout:smithi038.front.sepia.ceph.com
2020-11-12T17:29:50.516 INFO:teuthology.orchestra.run.smithi038:> grep smithi038.front.sepia.ceph.com /etc/hosts
2020-11-12T17:29:50.566 INFO:teuthology.orchestra.run.smithi038.stdout:172.21.15.38 smithi038.front.sepia.ceph.com smithi038
2020-11-12T17:29:50.570 INFO:teuthology.orchestra.run.smithi038:> sudo hostname smithi038
2020-11-12T17:29:50.680 INFO:teuthology.orchestra.run.smithi038:> sudo sed -i -e 's/smithi038.front.sepia.ceph.com/smithi038/g' /etc/hosts
2020-11-12T17:29:50.713 INFO:teuthology.orchestra.run.smithi038:> sudo sed -i -e 's/172.21.15.38/172.21.15.38/g' /etc/hosts
2020-11-12T17:29:50.785 INFO:teuthology.orchestra.run.smithi038:> sudo sed -i -e 's/smithi038.front.sepia.ceph.com/smithi038/g' /etc/hostname
2020-11-12T17:29:50.855 INFO:teuthology.orchestra.run.smithi038:> sudo hostname smithi038
2020-11-12T17:29:50.925 INFO:teuthology.provision.fog.smithi038:Deploy complete!
2020-11-12T17:29:50.961 DEBUG:teuthology.parallel:result is None
2020-11-12T17:29:54.235 INFO:teuthology.orchestra.run:Running command with timeout 600
2020-11-12T17:29:54.236 INFO:teuthology.orchestra.run.smithi188:> while [ ! -e '/.cephlab_net_configured' ]; do sleep 5; done
2020-11-12T17:30:14.701 INFO:teuthology.provision.fog.smithi188:Node is ready
2020-11-12T17:30:14.702 INFO:teuthology.orchestra.run.smithi188:> hostname
2020-11-12T17:30:14.721 INFO:teuthology.orchestra.run.smithi188.stdout:smithi188.front.sepia.ceph.com
2020-11-12T17:30:14.721 INFO:teuthology.orchestra.run.smithi188:> grep smithi188.front.sepia.ceph.com /etc/hosts
2020-11-12T17:30:14.777 INFO:teuthology.orchestra.run.smithi188.stdout:172.21.15.188 smithi188.front.sepia.ceph.com smithi188
2020-11-12T17:30:14.777 INFO:teuthology.orchestra.run.smithi188:> sudo hostname smithi188
2020-11-12T17:30:14.912 INFO:teuthology.orchestra.run.smithi188:> sudo sed -i -e 's/smithi188.front.sepia.ceph.com/smithi188/g' /etc/hosts
2020-11-12T17:30:14.943 INFO:teuthology.orchestra.run.smithi188:> sudo sed -i -e 's/172.21.15.188/172.21.15.188/g' /etc/hosts
2020-11-12T17:30:15.012 INFO:teuthology.orchestra.run.smithi188:> sudo sed -i -e 's/smithi188.front.sepia.ceph.com/smithi188/g' /etc/hostname
2020-11-12T17:30:15.081 INFO:teuthology.orchestra.run.smithi188:> sudo hostname smithi188
2020-11-12T17:30:15.149 INFO:teuthology.provision.fog.smithi188:Deploy complete!
2020-11-12T17:30:15.152 DEBUG:teuthology.parallel:result is None
2020-11-12T17:30:17.171 INFO:teuthology.lock.ops:Checking smithi188.front.sepia.ceph.com
2020-11-12T17:30:17.173 INFO:teuthology.lock.ops:New key found. Updating...
2020-11-12T17:30:17.204 INFO:teuthology.lock.ops:Checking smithi038.front.sepia.ceph.com
2020-11-12T17:30:17.204 INFO:teuthology.lock.ops:New key found. Updating...
2020-11-12T17:30:17.225 INFO:teuthology.lock.ops:Updating [smithi188.front.sepia.ceph.com]: set os type and version on server
2020-11-12T17:30:17.227 INFO:teuthology.orchestra.remote:Trying to reconnect to host
2020-11-12T17:30:17.230 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi188.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:30:18.316 INFO:teuthology.orchestra.run.smithi188:> true
2020-11-12T17:30:18.480 INFO:teuthology.orchestra.run.smithi188:> uname -m
2020-11-12T17:30:18.534 INFO:teuthology.orchestra.run.smithi188.stdout:x86_64
2020-11-12T17:30:18.535 INFO:teuthology.orchestra.run.smithi188:> cat /etc/os-release
2020-11-12T17:30:18.588 INFO:teuthology.orchestra.run.smithi188.stdout:NAME="CentOS Linux"
2020-11-12T17:30:18.588 INFO:teuthology.orchestra.run.smithi188.stdout:VERSION="8 (Core)"
2020-11-12T17:30:18.588 INFO:teuthology.orchestra.run.smithi188.stdout:ID="centos"
2020-11-12T17:30:18.588 INFO:teuthology.orchestra.run.smithi188.stdout:ID_LIKE="rhel fedora"
2020-11-12T17:30:18.589 INFO:teuthology.orchestra.run.smithi188.stdout:VERSION_ID="8"
2020-11-12T17:30:18.589 INFO:teuthology.orchestra.run.smithi188.stdout:PLATFORM_ID="platform:el8"
2020-11-12T17:30:18.589 INFO:teuthology.orchestra.run.smithi188.stdout:PRETTY_NAME="CentOS Linux 8 (Core)"
2020-11-12T17:30:18.589 INFO:teuthology.orchestra.run.smithi188.stdout:ANSI_COLOR="0;31"
2020-11-12T17:30:18.589 INFO:teuthology.orchestra.run.smithi188.stdout:CPE_NAME="cpe:/o:centos:centos:8"
2020-11-12T17:30:18.589 INFO:teuthology.orchestra.run.smithi188.stdout:HOME_URL="https://www.centos.org/"
2020-11-12T17:30:18.589 INFO:teuthology.orchestra.run.smithi188.stdout:BUG_REPORT_URL="https://bugs.centos.org/"
2020-11-12T17:30:18.590 INFO:teuthology.orchestra.run.smithi188.stdout:
2020-11-12T17:30:18.590 INFO:teuthology.orchestra.run.smithi188.stdout:CENTOS_MANTISBT_PROJECT="CentOS-8"
2020-11-12T17:30:18.590 INFO:teuthology.orchestra.run.smithi188.stdout:CENTOS_MANTISBT_PROJECT_VERSION="8"
2020-11-12T17:30:18.590 INFO:teuthology.orchestra.run.smithi188.stdout:REDHAT_SUPPORT_PRODUCT="centos"
2020-11-12T17:30:18.590 INFO:teuthology.orchestra.run.smithi188.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8"
2020-11-12T17:30:18.590 INFO:teuthology.orchestra.run.smithi188.stdout:
2020-11-12T17:30:18.630 INFO:teuthology.lock.ops:Updating smithi188.front.sepia.ceph.com on lock server
2020-11-12T17:30:18.651 INFO:teuthology.lock.ops:Updating [smithi038.front.sepia.ceph.com]: set os type and version on server
2020-11-12T17:30:18.652 INFO:teuthology.orchestra.remote:Trying to reconnect to host
2020-11-12T17:30:18.652 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi038.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-12T17:30:18.731 INFO:teuthology.orchestra.run.smithi038:> true
2020-11-12T17:30:18.908 INFO:teuthology.orchestra.run.smithi038:> uname -m
2020-11-12T17:30:18.963 INFO:teuthology.orchestra.run.smithi038.stdout:x86_64
2020-11-12T17:30:18.963 INFO:teuthology.orchestra.run.smithi038:> cat /etc/os-release
2020-11-12T17:30:19.017 INFO:teuthology.orchestra.run.smithi038.stdout:NAME="CentOS Linux"
2020-11-12T17:30:19.018 INFO:teuthology.orchestra.run.smithi038.stdout:VERSION="8 (Core)"
2020-11-12T17:30:19.018 INFO:teuthology.orchestra.run.smithi038.stdout:ID="centos"
2020-11-12T17:30:19.018 INFO:teuthology.orchestra.run.smithi038.stdout:ID_LIKE="rhel fedora"
2020-11-12T17:30:19.018 INFO:teuthology.orchestra.run.smithi038.stdout:VERSION_ID="8"
2020-11-12T17:30:19.018 INFO:teuthology.orchestra.run.smithi038.stdout:PLATFORM_ID="platform:el8"
2020-11-12T17:30:19.019 INFO:teuthology.orchestra.run.smithi038.stdout:PRETTY_NAME="CentOS Linux 8 (Core)"
2020-11-12T17:30:19.019 INFO:teuthology.orchestra.run.smithi038.stdout:ANSI_COLOR="0;31"
2020-11-12T17:30:19.019 INFO:teuthology.orchestra.run.smithi038.stdout:CPE_NAME="cpe:/o:centos:centos:8"
2020-11-12T17:30:19.019 INFO:teuthology.orchestra.run.smithi038.stdout:HOME_URL="https://www.centos.org/"
2020-11-12T17:30:19.019 INFO:teuthology.orchestra.run.smithi038.stdout:BUG_REPORT_URL="https://bugs.centos.org/"
2020-11-12T17:30:19.019 INFO:teuthology.orchestra.run.smithi038.stdout:
2020-11-12T17:30:19.019 INFO:teuthology.orchestra.run.smithi038.stdout:CENTOS_MANTISBT_PROJECT="CentOS-8"
2020-11-12T17:30:19.020 INFO:teuthology.orchestra.run.smithi038.stdout:CENTOS_MANTISBT_PROJECT_VERSION="8"
2020-11-12T17:30:19.020 INFO:teuthology.orchestra.run.smithi038.stdout:REDHAT_SUPPORT_PRODUCT="centos"
2020-11-12T17:30:19.020 INFO:teuthology.orchestra.run.smithi038.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8"
2020-11-12T17:30:19.020 INFO:teuthology.orchestra.run.smithi038.stdout:
2020-11-12T17:30:19.021 INFO:teuthology.lock.ops:Updating smithi038.front.sepia.ceph.com on lock server
2020-11-12T17:30:19.040 INFO:teuthology.task.internal.lock_machines:2 smithi machines locked this try, 2/2 locked so far
2020-11-12T17:30:19.628 INFO:teuthology.task.internal.lock_machines:Locked targets:
  smithi038.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC7o3swMHLuLPs2Ydb+Dorg/EftFzK70Wvz5xuoMzeXDRN/XHy5UzV08c4iNJUC+IixQED9nDc2SExjUWOS5Q8OCHCcg6trnkD4JDIVXESwfTjE+vP0r5tDObVYGZ/a7lgLlU3/eGnC8SywhVbYmWW8HpWy/MPe+hXEGIUxt0RUoJuXvrUWaWp/tjTv03q8/OnLLHrpFxvKiEwsAN7qm1a7IlayW33cDOfEffMidgAWsRFthShR50e5Vi0u6ZofaCJtT6OmNkr7AmWpYuSSDQCMdO0qHlGvWOtY6lMB3+b5Ax4gEIfyXeCCKLJ3h18yHQZh5HAJO38D25MA5WIjAjy3qKvBQYRr2u/yWZv8bHXGZLgek8NP3ehOTyi8wIvezOMkc/WkrLB+y9uHSL5aaWk1+CvrP5L8hyo/B0KXt3AalMvRkdVvvVgtO43xpEB2rxPnfyIr4dIQhfwy9qNqOd1uxMeAh7Zq958+ghNOMu+MiasCbkhOD+q1OYV7aj5NiYU=
  smithi188.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDHRw2KZNRtaSzQC5o7rSBigvniujBuIIr6s6PRkIT3l/4RLnRvftBA0a15KTjJ8bglsrqBvQY2HkGgHdbDF2a2rnXQiyutoH5dG7LT2sahgChBCCbKIa+3HmttAOQs1a3a0pKmwJlm1wZGnqNhKTM/gEug7fUAZAcPc1q0FnoaU3tmH8tk+DLtFgFIg2GpflEWLgYfmLbfU+trWknFCoaGwSawk4REf5Ygbp1nVChjYnXFPo4b2aKxcNKwh30Q4a1LTVFTv5BjKmAvAfuzJlUllYSpoFRO8tsUVcyH+RM0R9Ci65MQ8EA8cqgv2CLlgHRE5zE/c+vTe6nXOJ0f9Aq1tMY7LhhBw0hCprivCllePybLYrA3L5Ua+V4fX3BNmCS38YWGJrthm1Bkdx9vnyR6bjqhGuyW1pEB6NtZHbwVL0v1XvCAO2I6m1weJNRbxnTXDFEY2reMFVkpqHxZfQ+DflSEAIGHyntqziZBLCKLRi29dA9SrkIOJzjiRzCgmJM=
2020-11-12T17:30:19.629 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-11-12T17:30:19.698 INFO:teuthology.run_tasks:Running task internal.save_config...
2020-11-12T17:30:19.725 INFO:teuthology.task.internal:Saving configuration
2020-11-12T17:30:19.746 INFO:teuthology.run_tasks:Running task internal.check_lock...
2020-11-12T17:30:19.755 INFO:teuthology.task.internal.check_lock:Checking locks...
2020-11-12T17:30:19.771 DEBUG:teuthology.task.internal.check_lock:machine status is {'is_vm': False, 'locked': True, 'locked_since': '2020-11-12 17:25:19.556031', 'description': '/home/teuthworker/archive/yuriw-2020-11-12_17:03:16-rados-octopus-distro-basic-smithi/5616044', 'locked_by': 'scheduled_yuriw@teuthology', 'up': True, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCekRAX/o9If6gv3JGGgXCpiA6XcehFW5CFQ8IozFmGt7tffw8GRheMCYDDgGNhxYeYJ1Wwo1v+WR5BVPmgAHj4=', 'os_version': '8', 'machine_type': 'smithi', 'vm_host': None, 'os_type': 'centos', 'arch': 'x86_64', 'mac_address': None, 'name': 'smithi188.front.sepia.ceph.com'}
2020-11-12T17:30:19.787 DEBUG:teuthology.task.internal.check_lock:machine status is {'is_vm': False, 'locked': True, 'locked_since': '2020-11-12 17:25:19.556052', 'description': '/home/teuthworker/archive/yuriw-2020-11-12_17:03:16-rados-octopus-distro-basic-smithi/5616044', 'locked_by': 'scheduled_yuriw@teuthology', 'up': True, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBErmBrDkw9LcoY1ZrYmSSGtx8i+UayNmYNST3PzzBr0opfeV44hkn9GGdFGJTws3Qzibp9KV0SbUpx+51Ibr1/U=', 'os_version': '8', 'machine_type': 'smithi', 'vm_host': None, 'os_type': 'centos', 'arch': 'x86_64', 'mac_address': None, 'name': 'smithi038.front.sepia.ceph.com'}
2020-11-12T17:30:19.789 INFO:teuthology.run_tasks:Running task internal.add_remotes...
2020-11-12T17:30:19.801 INFO:teuthology.task.internal:roles: ubuntu@smithi188.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a']
2020-11-12T17:30:19.801 INFO:teuthology.task.internal:roles: ubuntu@smithi038.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'node-exporter.b']
2020-11-12T17:30:19.801 INFO:teuthology.run_tasks:Running task console_log...
2020-11-12T17:30:21.045 DEBUG:teuthology.exit:Got signal 15; running 1 handler...
2020-11-12T17:30:21.052 DEBUG:teuthology.task.console_log:Killing console logger for smithi188
2020-11-12T17:30:21.064 ERROR:teuthology.run_tasks:Saw exception from tasks.
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/run_tasks.py", line 90, in run_tasks
    manager = run_one_task(taskname, ctx=ctx, config=config)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/run_tasks.py", line 69, in run_one_task
    return task(**kwargs)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/task/console_log.py", line 17, in __init__
    super(ConsoleLog, self).__init__(ctx, config)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/task/__init__.py", line 36, in __init__
    self.filter_hosts()
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/task/console_log.py", line 33, in filter_hosts
    if not hasattr(remote.console, 'spawn_sol_log'):
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/remote.py", line 645, in console
    self._console = getRemoteConsole(self.name)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/remote.py", line 687, in getRemoteConsole
    name, ipmiuser, ipmipass, ipmidomain, logfile, timeout)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/console.py", line 55, in __init__
    stderr=subprocess.STDOUT).wait() == 0
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/psutil/__init__.py", line 1430, in __init__
    self.__subproc = subprocess.Popen(*args, **kwargs)
  File "/usr/lib/python3.6/subprocess.py", line 729, in __init__
    restore_signals, start_new_session)
  File "/usr/lib/python3.6/subprocess.py", line 1295, in _execute_child
    restore_signals, start_new_session, preexec_fn)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/exit.py", line 48, in default_handler
    handler.func(signal_, frame)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/task/console_log.py", line 83, in kill_console_loggers
    proc.terminate()
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/psutil/__init__.py", line 392, in wrapper
    raise NoSuchProcess(self.pid, self._name)
psutil.NoSuchProcess: psutil.NoSuchProcess process no longer exists (pid=6594)