2020-11-16T20:11:58.561 INFO:root:teuthology version: 1.0.0-357fd4a
2020-11-16T20:11:58.563 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --lock --block --archive /home/teuthworker/archive/kchai-2020-11-16_12:11:03-rados-wip-kefu-testing-2020-11-16-1845-distro-basic-smithi/5631146 --description rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-async-partial-recovery} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/crush-compat mon_election/connectivity msgr-failures/osd-delay msgr/async objectstore/bluestore-stupid rados supported-random-distro$/{ubuntu_latest} thrashers/careful thrashosds-health workloads/cache-pool-snaps} --owner scheduled_kchai@teuthology --name kchai-2020-11-16_12:11:03-rados-wip-kefu-testing-2020-11-16-1845-distro-basic-smithi -- /tmp/teuthology-worker.qrw_i0gy.tmp
2020-11-16T20:11:58.608 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-11-16T20:11:58.691 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/kchai-2020-11-16_12:11:03-rados-wip-kefu-testing-2020-11-16-1845-distro-basic-smithi/5631146
  branch: wip-kefu-testing-2020-11-16-1845
  description: rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log
    2-recovery-overrides/{more-async-partial-recovery} backoff/peering_and_degraded
    ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/crush-compat mon_election/connectivity
    msgr-failures/osd-delay msgr/async objectstore/bluestore-stupid rados supported-random-distro$/{ubuntu_latest}
    thrashers/careful thrashosds-health workloads/cache-pool-snaps}
  email: tchaikov@gmail.com
  first_in_suite: false
  job_id: '5631146'
  kernel:
    kdb: true
    sha1: distro
  last_in_suite: false
  machine_type: smithi
  name: kchai-2020-11-16_12:11:03-rados-wip-kefu-testing-2020-11-16-1845-distro-basic-smithi
  nuke-on-error: true
  openstack:
  - volumes:
      count: 4
      size: 10
  - volumes:
      count: 4
      size: 30
  - machine:
      ram: 15000
  os_type: ubuntu
  os_version: '18.04'
  overrides:
    admin_socket:
      branch: wip-kefu-testing-2020-11-16-1845
    ceph:
      conf:
        global:
          mon client directed command retry: 5
          mon election default strategy: 3
          ms inject delay max: 1
          ms inject delay probability: 0.005
          ms inject delay type: osd
          ms inject internal delays: 0.002
          ms inject socket failures: 2500
          ms type: async
          osd_async_recovery_min_cost: 1
          osd_max_pg_log_entries: 2
          osd_min_pg_log_entries: 1
          osd_object_clean_region_max_num_intervals: 1000
          osd_pg_log_trim_min: 0
          osd_pool_default_min_size: 2
          osd_pool_default_size: 3
        mgr:
          debug mgr: 20
          debug ms: 1
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
          mon min osdmap epochs: 50
          mon osdmap full prune interval: 2
          mon osdmap full prune min: 15
          mon osdmap full prune txsize: 2
          paxos service trim min: 10
        osd:
          bdev async discard: true
          bdev enable discard: true
          bluestore allocator: stupid
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          debug bluefs: 20
          debug bluestore: 20
          debug filestore: 20
          debug journal: 20
          debug ms: 20
          debug osd: 25
          debug rocksdb: 10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd backoff on degraded: true
          osd backoff on peering: true
          osd debug reject backfill probability: 0.3
          osd debug verify cached snaps: true
          osd debug verify missing on start: true
          osd failsafe full ratio: 0.95
          osd max backfills: 3
          osd max markdown count: 1000
          osd objectstore: bluestore
          osd op queue: debug_random
          osd op queue cut off: debug_random
          osd scrub max interval: 120
          osd scrub min interval: 60
          osd shutdown pgref assert: true
          osd snap trim sleep: 2
          osd_max_pg_log_entries: 3000
          osd_min_pg_log_entries: 3000
      fs: xfs
      log-ignorelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      - Health detail
      - \(OSD_SLOW_PING_TIME
      - but it is still running
      - objects unfound and apparently lost
      - overall HEALTH_
      - \(OSDMAP_FLAGS\)
      - \(OSD_
      - \(PG_
      - \(POOL_
      - \(CACHE_POOL_
      - \(SMALLER_PGP_NUM\)
      - \(OBJECT_
      - \(SLOW_OPS\)
      - \(REQUEST_SLOW\)
      - \(TOO_FEW_PGS\)
      - slow request
      - must scrub before tier agent can activate
      log-whitelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      - Health detail
      sha1: 911cc35794d871071a8bebfb0919482263b1d18b
    ceph-deploy:
      bluestore: true
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon:
          osd default pool size: 2
        osd:
          bdev async discard: true
          bdev enable discard: true
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          debug bluefs: 20
          debug bluestore: 20
          debug rocksdb: 10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd failsafe full ratio: 0.95
          osd objectstore: bluestore
      fs: xfs
    install:
      ceph:
        sha1: 911cc35794d871071a8bebfb0919482263b1d18b
    thrashosds:
      bdev_inject_crash: 2
      bdev_inject_crash_probability: 0.5
      min_in: 4
    workunit:
      branch: wip-kefu-testing-2020-11-16-1845
      sha1: 911cc35794d871071a8bebfb0919482263b1d18b
  owner: scheduled_kchai@teuthology
  priority: 90
  repo: git://git.ceph.com/ceph-ci.git
  roles:
  - - mon.a
    - mon.c
    - mgr.y
    - osd.0
    - osd.1
    - osd.2
    - osd.3
    - client.0
    - node-exporter.a
  - - mon.b
    - mgr.x
    - osd.4
    - osd.5
    - osd.6
    - osd.7
    - client.1
    - prometheus.a
    - node-exporter.b
  sha1: 911cc35794d871071a8bebfb0919482263b1d18b
  sleep_before_teardown: 0
  suite: rados
  suite_branch: wip-kefu-testing-2020-11-16-1845
  suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_wip-kefu-testing-2020-11-16-1845/qa
  suite_relpath: qa
  suite_repo: git://git.ceph.com/ceph-ci.git
  suite_sha1: 911cc35794d871071a8bebfb0919482263b1d18b
  tasks:
  - install: null
  - ceph:
      conf:
        osd:
          debug monc: 20
  - exec:
      mon.a:
      - while ! ceph balancer status ; do sleep 1 ; done
      - ceph balancer mode crush-compat
      - ceph balancer on
  - thrashosds:
      aggressive_pg_num_changes: false
      chance_pgnum_grow: 1
      chance_pgnum_shrink: 1
      chance_pgpnum_fix: 1
      timeout: 1200
  - exec:
      client.0:
      - sudo ceph osd pool create base 4
      - sudo ceph osd pool application enable base rados
      - sudo ceph osd pool create cache 4
      - sudo ceph osd tier add base cache
      - sudo ceph osd tier cache-mode cache writeback
      - sudo ceph osd tier set-overlay base cache
      - sudo ceph osd pool set cache hit_set_type bloom
      - sudo ceph osd pool set cache hit_set_count 8
      - sudo ceph osd pool set cache hit_set_period 3600
      - sudo ceph osd pool set cache target_max_objects 250
      - sudo ceph osd pool set cache min_read_recency_for_promote 0
      - sudo ceph osd pool set cache min_write_recency_for_promote 0
  - rados:
      clients:
      - client.0
      objects: 500
      op_weights:
        cache_evict: 50
        cache_flush: 50
        cache_try_flush: 50
        copy_from: 50
        delete: 50
        read: 100
        rollback: 50
        snap_create: 50
        snap_remove: 50
        write: 100
      ops: 4000
      pool_snaps: true
      pools:
      - base
  teuthology_branch: master
  timestamp: 2020-11-16_12:11:03
  tube: smithi
  user: kchai
  verbose: false
  worker_log: /home/teuthworker/archive/worker_logs/worker.smithi.7051
2020-11-16T20:11:58.692 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_wip-kefu-testing-2020-11-16-1845/qa; will attempt to use it
2020-11-16T20:11:58.693 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_wip-kefu-testing-2020-11-16-1845/qa/tasks
2020-11-16T20:11:58.694 INFO:teuthology.run_tasks:Running task internal.check_packages...
2020-11-16T20:11:58.696 INFO:teuthology.task.internal:Checking packages...
2020-11-16T20:11:58.727 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'basic' and ceph hash '911cc35794d871071a8bebfb0919482263b1d18b'
2020-11-16T20:11:58.727 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2020-11-16T20:11:58.728 INFO:teuthology.packaging:ref: None
2020-11-16T20:11:58.729 INFO:teuthology.packaging:tag: None
2020-11-16T20:11:58.729 INFO:teuthology.packaging:branch: wip-kefu-testing-2020-11-16-1845
2020-11-16T20:11:58.730 INFO:teuthology.packaging:sha1: 911cc35794d871071a8bebfb0919482263b1d18b
2020-11-16T20:11:58.730 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F18.04%2Fx86_64&ref=wip-kefu-testing-2020-11-16-1845
2020-11-16T20:11:58.913 INFO:teuthology.task.internal:Found packages for ceph version 16.0.0-7219-g911cc357-1bionic
2020-11-16T20:11:58.914 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2020-11-16T20:11:58.923 INFO:teuthology.task.internal:no buildpackages task found
2020-11-16T20:11:58.924 INFO:teuthology.run_tasks:Running task internal.lock_machines...
2020-11-16T20:11:58.952 INFO:teuthology.task.internal.lock_machines:Locking machines...
2020-11-16T20:11:58.952 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-11-16T20:11:59.006 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:12:09.024 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:12:19.042 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:12:29.058 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:12:39.076 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:12:49.091 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:12:59.117 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:13:09.134 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:13:19.149 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:13:29.161 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:13:39.177 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:13:49.194 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:13:59.213 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:14:09.229 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:14:19.247 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:14:29.266 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:14:39.281 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:14:49.291 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:14:59.306 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:15:09.322 INFO:teuthology.task.internal.lock_machines:waiting for more smithi machines to be free (need 0 + 2, have 0)...
2020-11-16T20:15:19.339 DEBUG:teuthology.lock.ops:lock_many request: {'locked_by': 'scheduled_kchai@teuthology', 'count': 2, 'machine_type': 'smithi', 'description': '/home/teuthworker/archive/kchai-2020-11-16_12:11:03-rados-wip-kefu-testing-2020-11-16-1845-distro-basic-smithi/5631146'}
2020-11-16T20:15:19.360 DEBUG:teuthology.lock.ops:locked ubuntu@smithi191.front.sepia.ceph.com, ubuntu@smithi179.front.sepia.ceph.com
2020-11-16T20:15:35.985 DEBUG:teuthology.exit:Installing handler: Handler(exiter=<teuthology.exit.Exiter object at 0x7efd2b46fdd8>, func=<function ConsoleLog.start_logging.<locals>.kill_console_loggers at 0x7efd29d757b8>, signals=[15])
2020-11-16T20:15:36.180 INFO:teuthology.provision.fog.smithi179:Scheduling deploy of ubuntu 18.04
2020-11-16T20:15:36.190 INFO:teuthology.provision.fog.smithi191:Scheduling deploy of ubuntu 18.04
2020-11-16T20:15:44.685 INFO:teuthology.orchestra.console:Power off smithi179
2020-11-16T20:15:44.686 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi179.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2020-11-16T20:15:44.703 INFO:teuthology.orchestra.console:Power off smithi191
2020-11-16T20:15:44.704 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi191.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2020-11-16T20:15:44.714 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi179.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-16T20:15:44.727 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi191.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-16T20:15:48.738 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi179.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-16T20:15:48.749 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi191.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-16T20:15:52.852 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi179.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-16T20:16:13.218 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi191.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-16T20:16:13.420 INFO:teuthology.orchestra.console:Power off for smithi179 completed
2020-11-16T20:16:13.428 INFO:teuthology.orchestra.console:Power off for smithi191 completed
2020-11-16T20:16:13.522 INFO:teuthology.orchestra.console:Power on smithi179
2020-11-16T20:16:13.523 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi179.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power on
2020-11-16T20:16:13.533 INFO:teuthology.orchestra.console:Power on smithi191
2020-11-16T20:16:13.533 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi191.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power on
2020-11-16T20:16:13.545 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi179.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-16T20:16:13.554 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi191.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-16T20:16:17.566 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi179.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-16T20:16:21.663 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi191.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2020-11-16T20:16:21.865 INFO:teuthology.orchestra.console:Power on for smithi179 completed
2020-11-16T20:16:21.874 INFO:teuthology.orchestra.console:Power on for smithi191 completed
2020-11-16T20:16:21.967 INFO:teuthology.provision.fog.smithi179:Waiting for deploy to finish
2020-11-16T20:16:21.974 INFO:teuthology.provision.fog.smithi191:Waiting for deploy to finish
2020-11-16T20:19:23.094 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:19:23.097 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:19:47.511 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:19:47.513 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:19:56.594 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:19:56.596 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:20:05.653 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:20:05.654 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:20:14.740 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:20:14.741 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:21:20.747 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:21:20.748 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:21:20.835 INFO:teuthology.orchestra.run:Running command with timeout 600
2020-11-16T20:21:20.836 INFO:teuthology.orchestra.run.smithi191:> while [ ! -e '/.cephlab_net_configured' ]; do sleep 5; done
2020-11-16T20:21:20.913 INFO:teuthology.orchestra.run:Running command with timeout 600
2020-11-16T20:21:20.914 INFO:teuthology.orchestra.run.smithi179:> while [ ! -e '/.cephlab_net_configured' ]; do sleep 5; done
2020-11-16T20:21:21.203 INFO:teuthology.provision.fog.smithi191:Node is ready
2020-11-16T20:21:21.203 INFO:teuthology.orchestra.run.smithi191:> hostname
2020-11-16T20:21:21.252 INFO:teuthology.orchestra.run.smithi191.stdout:smithi191.front.sepia.ceph.com
2020-11-16T20:21:21.253 INFO:teuthology.orchestra.run.smithi191:> grep smithi191.front.sepia.ceph.com /etc/hosts
2020-11-16T20:21:21.287 INFO:teuthology.provision.fog.smithi179:Node is ready
2020-11-16T20:21:21.287 INFO:teuthology.orchestra.run.smithi179:> hostname
2020-11-16T20:21:21.302 INFO:teuthology.orchestra.run.smithi191.stdout:172.21.15.191 smithi191.front.sepia.ceph.com smithi191
2020-11-16T20:21:21.302 INFO:teuthology.orchestra.run.smithi191:> sudo hostname smithi191
2020-11-16T20:21:21.335 INFO:teuthology.orchestra.run.smithi179.stdout:smithi179.front.sepia.ceph.com
2020-11-16T20:21:21.336 INFO:teuthology.orchestra.run.smithi179:> grep smithi179.front.sepia.ceph.com /etc/hosts
2020-11-16T20:21:21.383 INFO:teuthology.orchestra.run.smithi191:> sudo sed -i -e 's/smithi191.front.sepia.ceph.com/smithi191/g' /etc/hosts
2020-11-16T20:21:21.384 INFO:teuthology.orchestra.run.smithi179.stdout:172.21.15.179 smithi179.front.sepia.ceph.com smithi179
2020-11-16T20:21:21.384 INFO:teuthology.orchestra.run.smithi179:> sudo hostname smithi179
2020-11-16T20:21:21.438 INFO:teuthology.orchestra.run.smithi191:> sudo sed -i -e 's/172.21.15.191/172.21.15.191/g' /etc/hosts
2020-11-16T20:21:21.493 INFO:teuthology.orchestra.run.smithi191:> sudo sed -i -e 's/smithi191.front.sepia.ceph.com/smithi191/g' /etc/hostname
2020-11-16T20:21:21.549 INFO:teuthology.orchestra.run.smithi179:> sudo sed -i -e 's/smithi179.front.sepia.ceph.com/smithi179/g' /etc/hosts
2020-11-16T20:21:21.551 INFO:teuthology.orchestra.run.smithi191:> sudo hostname smithi191
2020-11-16T20:21:21.563 INFO:teuthology.orchestra.run.smithi179:> sudo sed -i -e 's/172.21.15.179/172.21.15.179/g' /etc/hosts
2020-11-16T20:21:21.605 INFO:teuthology.provision.fog.smithi191:Deploy complete!
2020-11-16T20:21:21.606 DEBUG:teuthology.parallel:result is None
2020-11-16T20:21:21.621 INFO:teuthology.orchestra.run.smithi179:> sudo sed -i -e 's/smithi179.front.sepia.ceph.com/smithi179/g' /etc/hostname
2020-11-16T20:21:21.677 INFO:teuthology.orchestra.run.smithi179:> sudo hostname smithi179
2020-11-16T20:21:21.731 INFO:teuthology.provision.fog.smithi179:Deploy complete!
2020-11-16T20:21:21.732 DEBUG:teuthology.parallel:result is None
2020-11-16T20:21:21.885 INFO:teuthology.lock.ops:Checking smithi191.front.sepia.ceph.com
2020-11-16T20:21:21.886 INFO:teuthology.lock.ops:New key found. Updating...
2020-11-16T20:21:21.912 INFO:teuthology.lock.ops:Checking smithi179.front.sepia.ceph.com
2020-11-16T20:21:21.912 INFO:teuthology.lock.ops:New key found. Updating...
2020-11-16T20:21:21.926 INFO:teuthology.lock.ops:Updating [smithi191.front.sepia.ceph.com]: set os type and version on server
2020-11-16T20:21:21.927 INFO:teuthology.orchestra.remote:Trying to reconnect to host
2020-11-16T20:21:21.927 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:21:21.998 INFO:teuthology.orchestra.run.smithi191:> true
2020-11-16T20:21:22.175 INFO:teuthology.orchestra.run.smithi191:> uname -m
2020-11-16T20:21:22.221 INFO:teuthology.orchestra.run.smithi191.stdout:x86_64
2020-11-16T20:21:22.222 INFO:teuthology.orchestra.run.smithi191:> cat /etc/os-release
2020-11-16T20:21:22.274 INFO:teuthology.orchestra.run.smithi191.stdout:NAME="Ubuntu"
2020-11-16T20:21:22.275 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION="18.04.5 LTS (Bionic Beaver)"
2020-11-16T20:21:22.275 INFO:teuthology.orchestra.run.smithi191.stdout:ID=ubuntu
2020-11-16T20:21:22.275 INFO:teuthology.orchestra.run.smithi191.stdout:ID_LIKE=debian
2020-11-16T20:21:22.275 INFO:teuthology.orchestra.run.smithi191.stdout:PRETTY_NAME="Ubuntu 18.04.5 LTS"
2020-11-16T20:21:22.275 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION_ID="18.04"
2020-11-16T20:21:22.275 INFO:teuthology.orchestra.run.smithi191.stdout:HOME_URL="https://www.ubuntu.com/"
2020-11-16T20:21:22.275 INFO:teuthology.orchestra.run.smithi191.stdout:SUPPORT_URL="https://help.ubuntu.com/"
2020-11-16T20:21:22.276 INFO:teuthology.orchestra.run.smithi191.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
2020-11-16T20:21:22.276 INFO:teuthology.orchestra.run.smithi191.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
2020-11-16T20:21:22.276 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION_CODENAME=bionic
2020-11-16T20:21:22.276 INFO:teuthology.orchestra.run.smithi191.stdout:UBUNTU_CODENAME=bionic
2020-11-16T20:21:22.277 INFO:teuthology.lock.ops:Updating smithi191.front.sepia.ceph.com on lock server
2020-11-16T20:21:22.297 INFO:teuthology.lock.ops:Updating [smithi179.front.sepia.ceph.com]: set os type and version on server
2020-11-16T20:21:22.298 INFO:teuthology.orchestra.remote:Trying to reconnect to host
2020-11-16T20:21:22.299 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:21:22.376 INFO:teuthology.orchestra.run.smithi179:> true
2020-11-16T20:21:22.611 INFO:teuthology.orchestra.run.smithi179:> uname -m
2020-11-16T20:21:22.659 INFO:teuthology.orchestra.run.smithi179.stdout:x86_64
2020-11-16T20:21:22.660 INFO:teuthology.orchestra.run.smithi179:> cat /etc/os-release
2020-11-16T20:21:22.707 INFO:teuthology.orchestra.run.smithi179.stdout:NAME="Ubuntu"
2020-11-16T20:21:26.930 INFO:teuthology.orchestra.run.smithi179.stdout:VERSION="18.04.5 LTS (Bionic Beaver)"
2020-11-16T20:21:26.930 INFO:teuthology.orchestra.run.smithi179.stdout:ID=ubuntu
2020-11-16T20:21:26.930 INFO:teuthology.orchestra.run.smithi179.stdout:ID_LIKE=debian
2020-11-16T20:21:26.931 INFO:teuthology.orchestra.run.smithi179.stdout:PRETTY_NAME="Ubuntu 18.04.5 LTS"
2020-11-16T20:21:26.931 INFO:teuthology.orchestra.run.smithi179.stdout:VERSION_ID="18.04"
2020-11-16T20:21:26.931 INFO:teuthology.orchestra.run.smithi179.stdout:HOME_URL="https://www.ubuntu.com/"
2020-11-16T20:21:26.931 INFO:teuthology.orchestra.run.smithi179.stdout:SUPPORT_URL="https://help.ubuntu.com/"
2020-11-16T20:21:26.931 INFO:teuthology.orchestra.run.smithi179.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
2020-11-16T20:21:26.931 INFO:teuthology.orchestra.run.smithi179.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
2020-11-16T20:21:26.931 INFO:teuthology.orchestra.run.smithi179.stdout:VERSION_CODENAME=bionic
2020-11-16T20:21:26.932 INFO:teuthology.orchestra.run.smithi179.stdout:UBUNTU_CODENAME=bionic
2020-11-16T20:21:26.932 INFO:teuthology.lock.ops:Updating smithi179.front.sepia.ceph.com on lock server
2020-11-16T20:21:26.953 INFO:teuthology.task.internal.lock_machines:2 smithi machines locked this try, 2/2 locked so far
2020-11-16T20:21:27.002 INFO:teuthology.task.internal.lock_machines:Locked targets:
  smithi179.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDOlnntGsN1gm8q7+UnLvID2G/XwQiOKd96UOm7gmenk1U9kyrzvBKXEG/AZB/S82FGFS84Ya+aQ6xYbrV5r18ngN3D8rf5QOqU7zIDTyW/elsrCQ7el/bq7n+YlIxUvY3KErmgILgiQnSuvbKemERIKfJeFxH05U53Ol9cbZdZ9wKi+Y8YHAY0OV6KK6PDSRa102tC9+o1JRgZYdhAdX7eVl/CXUpoQKMkYA4eFjiOM18kN5mj2oIo+Je1LA7Gb+uv8xV8Q7ENKdokPXYi6eSQYgoMF1/xrVWz/7InwgSSueQTgDuqV7JNh4ZzAH9hx4y4eYvcjuxlauVqI9wOL6nx
  smithi191.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDOOws4SjlJHys1tBBsZKZmUhgbVaziMe9kgDZCD72S59O1ZfaRdUsGu0oChI/zfDfvqsvAvm5j1c45fnAlwDY48r1V2VjekmyAisO2oWhhPwKC2mJ+jrhknZqwr95O7Kng6fchXwZpcXloPyzw2JY4nHdKfSwJaePtx5256x0D/YLUxyc/IkAVG/0KT2rlxOCRjPw3jmyVsPD1I7j9eO7uM2BUfcqERZxjuvb1KfbUN0qbXoOx3OcNx5DspTl+4xrI/os3GsVEyuX6b5qDixB0qHdvjxOWVEsNruB/RpnaAF6SimiU43qYFuGviiuRzjonraR9T+InCKKbQa5kupzf
2020-11-16T20:21:27.003 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2020-11-16T20:21:27.065 INFO:teuthology.run_tasks:Running task internal.save_config...
2020-11-16T20:21:27.074 INFO:teuthology.task.internal:Saving configuration
2020-11-16T20:21:27.092 INFO:teuthology.run_tasks:Running task internal.check_lock...
2020-11-16T20:21:27.115 INFO:teuthology.task.internal.check_lock:Checking locks...
2020-11-16T20:21:27.134 DEBUG:teuthology.task.internal.check_lock:machine status is {'is_vm': False, 'locked': True, 'locked_since': '2020-11-16 20:15:19.347405', 'description': '/home/teuthworker/archive/kchai-2020-11-16_12:11:03-rados-wip-kefu-testing-2020-11-16-1845-distro-basic-smithi/5631146', 'locked_by': 'scheduled_kchai@teuthology', 'up': True, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKyAG581XVxEm4oDf6xbmLYBNEkXgyX0rz8EyAWKo738', 'os_version': '18.04', 'machine_type': 'smithi', 'vm_host': None, 'os_type': 'ubuntu', 'arch': 'x86_64', 'mac_address': None, 'name': 'smithi191.front.sepia.ceph.com'}
2020-11-16T20:21:27.150 DEBUG:teuthology.task.internal.check_lock:machine status is {'is_vm': False, 'locked': True, 'locked_since': '2020-11-16 20:15:19.347427', 'description': '/home/teuthworker/archive/kchai-2020-11-16_12:11:03-rados-wip-kefu-testing-2020-11-16-1845-distro-basic-smithi/5631146', 'locked_by': 'scheduled_kchai@teuthology', 'up': True, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGItOha/jo+dFW+SI6jwZbHamw65gF8FXOWydgxCTL6U', 'os_version': '18.04', 'machine_type': 'smithi', 'vm_host': None, 'os_type': 'ubuntu', 'arch': 'x86_64', 'mac_address': None, 'name': 'smithi179.front.sepia.ceph.com'}
2020-11-16T20:21:27.151 INFO:teuthology.run_tasks:Running task internal.add_remotes...
2020-11-16T20:21:27.158 INFO:teuthology.task.internal:roles: ubuntu@smithi191.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a']
2020-11-16T20:21:27.158 INFO:teuthology.task.internal:roles: ubuntu@smithi179.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'node-exporter.b']
2020-11-16T20:21:27.159 INFO:teuthology.run_tasks:Running task console_log...
2020-11-16T20:21:27.224 DEBUG:teuthology.exit:Installing handler: Handler(exiter=<teuthology.exit.Exiter object at 0x7efd2b46fdd8>, func=<function ConsoleLog.start_logging.<locals>.kill_console_loggers at 0x7efd2d822a60>, signals=[15])
2020-11-16T20:21:27.224 INFO:teuthology.run_tasks:Running task internal.connect...
2020-11-16T20:21:27.232 INFO:teuthology.task.internal:Opening connections...
2020-11-16T20:21:27.233 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi191.front.sepia.ceph.com
2020-11-16T20:21:27.235 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:21:27.311 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi179.front.sepia.ceph.com
2020-11-16T20:21:27.313 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2020-11-16T20:21:27.381 INFO:teuthology.run_tasks:Running task internal.push_inventory...
2020-11-16T20:21:27.389 INFO:teuthology.orchestra.run.smithi191:> uname -m
2020-11-16T20:21:27.518 INFO:teuthology.orchestra.run.smithi191.stdout:x86_64
2020-11-16T20:21:27.518 INFO:teuthology.orchestra.run.smithi191:> cat /etc/os-release
2020-11-16T20:21:27.577 INFO:teuthology.orchestra.run.smithi191.stdout:NAME="Ubuntu"
2020-11-16T20:21:27.577 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION="18.04.5 LTS (Bionic Beaver)"
2020-11-16T20:21:27.577 INFO:teuthology.orchestra.run.smithi191.stdout:ID=ubuntu
2020-11-16T20:21:27.577 INFO:teuthology.orchestra.run.smithi191.stdout:ID_LIKE=debian
2020-11-16T20:21:27.578 INFO:teuthology.orchestra.run.smithi191.stdout:PRETTY_NAME="Ubuntu 18.04.5 LTS"
2020-11-16T20:21:27.578 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION_ID="18.04"
2020-11-16T20:21:27.578 INFO:teuthology.orchestra.run.smithi191.stdout:HOME_URL="https://www.ubuntu.com/"
2020-11-16T20:21:27.578 INFO:teuthology.orchestra.run.smithi191.stdout:SUPPORT_URL="https://help.ubuntu.com/"
2020-11-16T20:21:27.579 INFO:teuthology.orchestra.run.smithi191.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
2020-11-16T20:21:27.579 INFO:teuthology.orchestra.run.smithi191.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
2020-11-16T20:21:27.579 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION_CODENAME=bionic
2020-11-16T20:21:27.579 INFO:teuthology.orchestra.run.smithi191.stdout:UBUNTU_CODENAME=bionic
2020-11-16T20:21:27.580 INFO:teuthology.lock.ops:Updating smithi191.front.sepia.ceph.com on lock server
2020-11-16T20:21:27.600 INFO:teuthology.orchestra.run.smithi179:> uname -m
2020-11-16T20:21:27.607 INFO:teuthology.orchestra.run.smithi179.stdout:x86_64
2020-11-16T20:21:27.608 INFO:teuthology.orchestra.run.smithi179:> cat /etc/os-release
2020-11-16T20:21:27.659 INFO:teuthology.orchestra.run.smithi179.stdout:NAME="Ubuntu"
2020-11-16T20:21:27.660 INFO:teuthology.orchestra.run.smithi179.stdout:VERSION="18.04.5 LTS (Bionic Beaver)"
2020-11-16T20:21:27.661 INFO:teuthology.orchestra.run.smithi179.stdout:ID=ubuntu
2020-11-16T20:21:27.661 INFO:teuthology.orchestra.run.smithi179.stdout:ID_LIKE=debian
2020-11-16T20:21:27.662 INFO:teuthology.orchestra.run.smithi179.stdout:PRETTY_NAME="Ubuntu 18.04.5 LTS"
2020-11-16T20:21:27.662 INFO:teuthology.orchestra.run.smithi179.stdout:VERSION_ID="18.04"
2020-11-16T20:21:27.663 INFO:teuthology.orchestra.run.smithi179.stdout:HOME_URL="https://www.ubuntu.com/"
2020-11-16T20:21:27.663 INFO:teuthology.orchestra.run.smithi179.stdout:SUPPORT_URL="https://help.ubuntu.com/"
2020-11-16T20:21:27.664 INFO:teuthology.orchestra.run.smithi179.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
2020-11-16T20:21:27.664 INFO:teuthology.orchestra.run.smithi179.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
2020-11-16T20:21:27.664 INFO:teuthology.orchestra.run.smithi179.stdout:VERSION_CODENAME=bionic
2020-11-16T20:21:27.665 INFO:teuthology.orchestra.run.smithi179.stdout:UBUNTU_CODENAME=bionic
2020-11-16T20:21:27.665 INFO:teuthology.lock.ops:Updating smithi179.front.sepia.ceph.com on lock server
2020-11-16T20:21:27.696 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles...