2023-03-25T21:36:57.342 INFO:root:teuthology version: 0.0.1.dev1+ge372916
2023-03-25T21:36:57.343 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_lflores@teuthology --verbose --description rados/thrash-erasure-code-overwrites/{bluestore-bitmap ceph clusters/{fixed-2 openstack} fast/fast mon_election/classic msgr-failures/fastclose rados recovery-overrides/{more-active-recovery} supported-random-distro$/{rhel_8} thrashers/fastread thrashosds-health workloads/ec-snaps-few-objects-overwrites} --name lflores-2023-03-25_21:26:30-rados-wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a-distro-default-smithi --archive /home/teuthworker/archive/lflores-2023-03-25_21:26:30-rados-wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a-distro-default-smithi/7219689 -- /home/teuthworker/archive/lflores-2023-03-25_21:26:30-rados-wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a-distro-default-smithi/7219689/orig.config.yaml
2023-03-25T21:36:57.389 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2023-03-25T21:36:57.468 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/lflores-2023-03-25_21:26:30-rados-wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a-distro-default-smithi/7219689
  branch: wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a
  description: rados/thrash-erasure-code-overwrites/{bluestore-bitmap ceph clusters/{fixed-2
    openstack} fast/fast mon_election/classic msgr-failures/fastclose rados recovery-overrides/{more-active-recovery}
    supported-random-distro$/{rhel_8} thrashers/fastread thrashosds-health workloads/ec-snaps-few-objects-overwrites}
  email: null
  first_in_suite: false
  job_id: '7219689'
  kernel:
    kdb: true
    sha1: distro
  last_in_suite: false
  machine_type: smithi
  name: lflores-2023-03-25_21:26:30-rados-wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a-distro-default-smithi
  no_nested_subset: false
  nuke-on-error: true
  openstack:
  - volumes:
      count: 4
      size: 10
  - volumes:
      count: 4
      size: 30
  os_type: rhel
  os_version: '8.6'
  overrides:
    admin_socket:
      branch: wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a
    ceph:
      conf:
        global:
          enable experimental unrecoverable data corrupting features: '*'
          mon client directed command retry: 5
          mon election default strategy: 1
          ms inject socket failures: 2500
          ms tcp read timeout: 5
          osd pool default ec fast read: true
          osd_recovery_max_active: 10
          osd_recovery_max_single_start: 10
        mgr:
          debug mgr: 20
          debug ms: 1
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
          mon scrub interval: 300
          osd pool default ec fast read: true
        osd:
          bdev async discard: true
          bdev enable discard: true
          bluestore allocator: bitmap
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          bluestore zero block detection: true
          debug bluefs: 20
          debug bluestore: 20
          debug ms: 1
          debug osd: 20
          debug rocksdb: 10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd blocked scrub grace period: 3600
          osd debug reject backfill probability: 0.1
          osd debug verify cached snaps: true
          osd debug verify missing on start: true
          osd failsafe full ratio: 0.95
          osd max backfills: 3
          osd max markdown count: 1000
          osd mclock override recovery settings: true
          osd objectstore: bluestore
          osd op queue: debug_random
          osd op queue cut off: debug_random
          osd scrub max interval: 120
          osd scrub min interval: 60
          osd shutdown pgref assert: true
      flavor: default
      fs: xfs
      log-ignorelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      - \(OSD_SLOW_PING_TIME
      - but it is still running
      - objects unfound and apparently lost
      - overall HEALTH_
      - \(OSDMAP_FLAGS\)
      - \(OSD_
      - \(PG_
      - \(POOL_
      - \(CACHE_POOL_
      - \(SMALLER_PGP_NUM\)
      - \(OBJECT_
      - \(SLOW_OPS\)
      - \(REQUEST_SLOW\)
      - \(TOO_FEW_PGS\)
      - slow request
      - timeout on replica
      - late reservation from
      sha1: 5e717292106ca2d310770101bfebb345837be8e1
    ceph-deploy:
      bluestore: true
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon:
          osd default pool size: 2
        osd:
          bdev async discard: true
          bdev enable discard: true
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          debug bluefs: 20
          debug bluestore: 20
          debug rocksdb: 10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd failsafe full ratio: 0.95
          osd objectstore: bluestore
      fs: xfs
    install:
      ceph:
        flavor: default
        sha1: 5e717292106ca2d310770101bfebb345837be8e1
    selinux:
      whitelist:
      - scontext=system_u:system_r:logrotate_t:s0
    thrashosds:
      bdev_inject_crash: 2
      bdev_inject_crash_probability: 0.5
      disable_objectstore_tool_tests: true
    workunit:
      branch: wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a
      sha1: 5e717292106ca2d310770101bfebb345837be8e1
  owner: scheduled_lflores@teuthology
  priority: 50
  repo: https://git.ceph.com/ceph-ci.git
  roles:
  - - mon.a
    - mon.c
    - mgr.y
    - osd.0
    - osd.1
    - osd.2
    - osd.3
    - client.0
    - node-exporter.a
  - - mon.b
    - mgr.x
    - osd.4
    - osd.5
    - osd.6
    - osd.7
    - client.1
    - prometheus.a
    - node-exporter.b
  sha1: 5e717292106ca2d310770101bfebb345837be8e1
  sleep_before_teardown: 0
  suite: rados
  suite_branch: wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a
  suite_path: /home/teuthworker/src/github.com_ljflores_ceph_5e717292106ca2d310770101bfebb345837be8e1/qa
  suite_relpath: qa
  suite_repo: https://github.com/ljflores/ceph.git
  suite_sha1: 5e717292106ca2d310770101bfebb345837be8e1
  targets:
    smithi062.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac=
    smithi086.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac=
  tasks:
  - install: null
  - ceph:
      conf:
        osd:
          debug monc: 20
  - thrashosds:
      chance_pgnum_grow: 1
      chance_pgnum_shrink: 1
      chance_pgpnum_fix: 1
      min_in: 4
      timeout: 1200
  - rados:
      clients:
      - client.0
      ec_pool: true
      erasure_code_use_overwrites: true
      objects: 50
      op_weights:
        copy_from: 50
        delete: 50
        read: 100
        rollback: 50
        snap_create: 50
        snap_remove: 50
        write: 100
      ops: 4000
  teuthology:
    fragments_dropped: []
    meta: {}
    postmerge: []
  teuthology_branch: main
  teuthology_sha1: e3729161eecf8e6d09a4069acbb883cdcd5e9f50
  timestamp: 2023-03-25_21:26:30
  tube: smithi
  user: lflores
  verbose: true
  worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.4538
2023-03-25T21:36:57.468 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ljflores_ceph_5e717292106ca2d310770101bfebb345837be8e1/qa; will attempt to use it
2023-03-25T21:36:57.469 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ljflores_ceph_5e717292106ca2d310770101bfebb345837be8e1/qa/tasks
2023-03-25T21:36:57.470 INFO:teuthology.run_tasks:Running task internal.check_packages...
2023-03-25T21:36:57.471 INFO:teuthology.task.internal:Checking packages...
2023-03-25T21:36:57.501 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash '5e717292106ca2d310770101bfebb345837be8e1'
2023-03-25T21:36:57.501 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2023-03-25T21:36:57.501 INFO:teuthology.packaging:ref: None
2023-03-25T21:36:57.502 INFO:teuthology.packaging:tag: None
2023-03-25T21:36:57.502 INFO:teuthology.packaging:branch: wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a
2023-03-25T21:36:57.502 INFO:teuthology.packaging:sha1: 5e717292106ca2d310770101bfebb345837be8e1
2023-03-25T21:36:57.502 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a
2023-03-25T21:36:57.641 INFO:teuthology.task.internal:Found packages for ceph version 18.0.0-3182.g5e717292
2023-03-25T21:36:57.642 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2023-03-25T21:36:57.650 INFO:teuthology.task.internal:no buildpackages task found
2023-03-25T21:36:57.651 INFO:teuthology.run_tasks:Running task internal.save_config...
2023-03-25T21:36:57.685 INFO:teuthology.task.internal:Saving configuration
2023-03-25T21:36:57.715 INFO:teuthology.run_tasks:Running task internal.check_lock...
2023-03-25T21:36:57.735 INFO:teuthology.task.internal.check_lock:Checking locks...
2023-03-25T21:36:57.764 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi062.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/lflores-2023-03-25_21:26:30-rados-wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a-distro-default-smithi/7219689', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-03-25 21:31:50.201999', 'locked_by': 'scheduled_lflores@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='}
2023-03-25T21:36:57.793 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi086.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/lflores-2023-03-25_21:26:30-rados-wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a-distro-default-smithi/7219689', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-03-25 21:31:50.204190', 'locked_by': 'scheduled_lflores@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='}
2023-03-25T21:36:57.793 INFO:teuthology.run_tasks:Running task internal.add_remotes...
2023-03-25T21:36:57.801 INFO:teuthology.task.internal:roles: ubuntu@smithi062.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a']
2023-03-25T21:36:57.801 INFO:teuthology.task.internal:roles: ubuntu@smithi086.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'node-exporter.b']
2023-03-25T21:36:57.801 INFO:teuthology.run_tasks:Running task console_log...
2023-03-25T21:36:57.906 DEBUG:teuthology.exit:Installing handler: Handler(exiter=<teuthology.exit.Exiter object at 0x7f93b362cfd0>, func=<function ConsoleLog.start_logging.<locals>.kill_console_loggers at 0x7f93b28ec310>, signals=[15])
2023-03-25T21:36:57.908 INFO:teuthology.run_tasks:Running task internal.connect...
2023-03-25T21:36:57.917 INFO:teuthology.task.internal:Opening connections...
2023-03-25T21:36:57.917 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi062.front.sepia.ceph.com
2023-03-25T21:36:57.919 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi062.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-03-25T21:36:57.993 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi086.front.sepia.ceph.com
2023-03-25T21:36:57.993 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi086.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-03-25T21:36:58.062 INFO:teuthology.run_tasks:Running task internal.push_inventory...
2023-03-25T21:36:58.071 DEBUG:teuthology.orchestra.run.smithi062:> uname -m
2023-03-25T21:36:58.092 INFO:teuthology.orchestra.run.smithi062.stdout:x86_64
2023-03-25T21:36:58.093 DEBUG:teuthology.orchestra.run.smithi062:> cat /etc/os-release
2023-03-25T21:36:58.152 INFO:teuthology.orchestra.run.smithi062.stdout:NAME="Red Hat Enterprise Linux"
2023-03-25T21:36:58.152 INFO:teuthology.orchestra.run.smithi062.stdout:VERSION="8.6 (Ootpa)"
2023-03-25T21:36:58.153 INFO:teuthology.orchestra.run.smithi062.stdout:ID="rhel"
2023-03-25T21:36:58.153 INFO:teuthology.orchestra.run.smithi062.stdout:ID_LIKE="fedora"
2023-03-25T21:36:58.153 INFO:teuthology.orchestra.run.smithi062.stdout:VERSION_ID="8.6"
2023-03-25T21:36:58.153 INFO:teuthology.orchestra.run.smithi062.stdout:PLATFORM_ID="platform:el8"
2023-03-25T21:36:58.153 INFO:teuthology.orchestra.run.smithi062.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)"
2023-03-25T21:36:58.153 INFO:teuthology.orchestra.run.smithi062.stdout:ANSI_COLOR="0;31"
2023-03-25T21:36:58.153 INFO:teuthology.orchestra.run.smithi062.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos"
2023-03-25T21:36:58.154 INFO:teuthology.orchestra.run.smithi062.stdout:HOME_URL="https://www.redhat.com/"
2023-03-25T21:36:58.154 INFO:teuthology.orchestra.run.smithi062.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/"
2023-03-25T21:36:58.154 INFO:teuthology.orchestra.run.smithi062.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/"
2023-03-25T21:36:58.154 INFO:teuthology.orchestra.run.smithi062.stdout:
2023-03-25T21:36:58.154 INFO:teuthology.orchestra.run.smithi062.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8"
2023-03-25T21:36:58.154 INFO:teuthology.orchestra.run.smithi062.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6
2023-03-25T21:36:58.154 INFO:teuthology.orchestra.run.smithi062.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
2023-03-25T21:36:58.155 INFO:teuthology.orchestra.run.smithi062.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6"
2023-03-25T21:36:58.155 INFO:teuthology.lock.ops:Updating smithi062.front.sepia.ceph.com on lock server
2023-03-25T21:36:58.176 DEBUG:teuthology.orchestra.run.smithi086:> uname -m
2023-03-25T21:36:58.197 INFO:teuthology.orchestra.run.smithi086.stdout:x86_64
2023-03-25T21:36:58.197 DEBUG:teuthology.orchestra.run.smithi086:> cat /etc/os-release
2023-03-25T21:36:58.258 INFO:teuthology.orchestra.run.smithi086.stdout:NAME="Red Hat Enterprise Linux"
2023-03-25T21:36:58.258 INFO:teuthology.orchestra.run.smithi086.stdout:VERSION="8.6 (Ootpa)"
2023-03-25T21:36:58.259 INFO:teuthology.orchestra.run.smithi086.stdout:ID="rhel"
2023-03-25T21:36:58.259 INFO:teuthology.orchestra.run.smithi086.stdout:ID_LIKE="fedora"
2023-03-25T21:36:58.259 INFO:teuthology.orchestra.run.smithi086.stdout:VERSION_ID="8.6"
2023-03-25T21:36:58.259 INFO:teuthology.orchestra.run.smithi086.stdout:PLATFORM_ID="platform:el8"
2023-03-25T21:36:58.259 INFO:teuthology.orchestra.run.smithi086.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)"
2023-03-25T21:36:58.259 INFO:teuthology.orchestra.run.smithi086.stdout:ANSI_COLOR="0;31"
2023-03-25T21:36:58.259 INFO:teuthology.orchestra.run.smithi086.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos"
2023-03-25T21:36:58.260 INFO:teuthology.orchestra.run.smithi086.stdout:HOME_URL="https://www.redhat.com/"
2023-03-25T21:36:58.260 INFO:teuthology.orchestra.run.smithi086.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/"
2023-03-25T21:36:58.260 INFO:teuthology.orchestra.run.smithi086.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/"
2023-03-25T21:36:58.260 INFO:teuthology.orchestra.run.smithi086.stdout:
2023-03-25T21:36:58.260 INFO:teuthology.orchestra.run.smithi086.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8"
2023-03-25T21:36:58.260 INFO:teuthology.orchestra.run.smithi086.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6
2023-03-25T21:36:58.260 INFO:teuthology.orchestra.run.smithi086.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
2023-03-25T21:36:58.260 INFO:teuthology.orchestra.run.smithi086.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6"
2023-03-25T21:36:58.261 INFO:teuthology.lock.ops:Updating smithi086.front.sepia.ceph.com on lock server
2023-03-25T21:36:58.285 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles...
2023-03-25T21:36:58.296 INFO:teuthology.run_tasks:Running task internal.check_conflict...
2023-03-25T21:36:58.327 INFO:teuthology.task.internal:Checking for old test directory...
2023-03-25T21:36:58.327 DEBUG:teuthology.orchestra.run.smithi062:> test '!' -e /home/ubuntu/cephtest
2023-03-25T21:36:58.330 DEBUG:teuthology.orchestra.run.smithi086:> test '!' -e /home/ubuntu/cephtest
2023-03-25T21:36:58.348 INFO:teuthology.run_tasks:Running task internal.check_ceph_data...
2023-03-25T21:36:58.391 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph...
2023-03-25T21:36:58.391 DEBUG:teuthology.orchestra.run.smithi062:> test -z $(ls -A /var/lib/ceph)
2023-03-25T21:36:58.394 DEBUG:teuthology.orchestra.run.smithi086:> test -z $(ls -A /var/lib/ceph)
2023-03-25T21:36:58.421 INFO:teuthology.run_tasks:Running task internal.vm_setup...
2023-03-25T21:36:58.485 INFO:teuthology.run_tasks:Running task kernel...
2023-03-25T21:36:58.503 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'}
2023-03-25T21:36:58.504 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300
2023-03-25T21:36:58.505 DEBUG:teuthology.orchestra.run.smithi062:> test -f /run/.containerenv -o -f /.dockerenv
2023-03-25T21:36:58.506 DEBUG:teuthology.orchestra.run.smithi086:> test -f /run/.containerenv -o -f /.dockerenv
2023-03-25T21:36:58.524 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-03-25T21:36:58.525 DEBUG:teuthology.orchestra.run.smithi086:> uname -r
2023-03-25T21:36:58.526 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-03-25T21:36:58.527 DEBUG:teuthology.orchestra.run.smithi062:> uname -r
2023-03-25T21:36:58.584 INFO:teuthology.orchestra.run.smithi086.stdout:4.18.0-372.9.1.el8.x86_64
2023-03-25T21:36:58.585 INFO:teuthology.task.kernel:Running kernel on smithi086: 4.18.0-372.9.1.el8.x86_64
2023-03-25T21:36:58.585 DEBUG:teuthology.orchestra.run.smithi086:> sudo yum install -y kernel
2023-03-25T21:36:58.586 INFO:teuthology.orchestra.run.smithi062.stdout:4.18.0-372.9.1.el8.x86_64
2023-03-25T21:36:58.587 INFO:teuthology.task.kernel:Running kernel on smithi062: 4.18.0-372.9.1.el8.x86_64
2023-03-25T21:36:58.587 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum install -y kernel
2023-03-25T21:36:59.807 INFO:teuthology.orchestra.run.smithi086.stdout:Updating Subscription Management repositories.
2023-03-25T21:36:59.808 INFO:teuthology.orchestra.run.smithi086.stdout:Unable to read consumer identity
2023-03-25T21:36:59.828 INFO:teuthology.orchestra.run.smithi062.stdout:Updating Subscription Management repositories.
2023-03-25T21:36:59.828 INFO:teuthology.orchestra.run.smithi062.stdout:Unable to read consumer identity
2023-03-25T21:37:01.267 INFO:teuthology.orchestra.run.smithi086.stdout:Copr repo for python3-asyncssh owned by ceph     20 kB/s | 3.5 kB     00:00
2023-03-25T21:37:01.321 INFO:teuthology.orchestra.run.smithi062.stdout:Copr repo for python3-asyncssh owned by ceph     19 kB/s | 3.5 kB     00:00
2023-03-25T21:37:02.559 INFO:teuthology.orchestra.run.smithi086.stdout:Extra Packages for Enterprise Linux              11 MB/s |  14 MB     00:01
2023-03-25T21:37:02.972 INFO:teuthology.orchestra.run.smithi062.stdout:Extra Packages for Enterprise Linux             8.4 MB/s |  14 MB     00:01
2023-03-25T21:37:05.676 INFO:teuthology.orchestra.run.smithi086.stdout:lab-extras                                      429 kB/s |  24 kB     00:00
2023-03-25T21:37:06.213 INFO:teuthology.orchestra.run.smithi062.stdout:lab-extras                                      409 kB/s |  24 kB     00:00
2023-03-25T21:37:06.361 INFO:teuthology.orchestra.run.smithi086.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed.
2023-03-25T21:37:06.414 INFO:teuthology.orchestra.run.smithi086.stdout:Dependencies resolved.
2023-03-25T21:37:06.414 INFO:teuthology.orchestra.run.smithi086.stdout:Nothing to do.
2023-03-25T21:37:06.415 INFO:teuthology.orchestra.run.smithi086.stdout:Complete!
2023-03-25T21:37:06.554 DEBUG:teuthology.orchestra.run.smithi086:> echo no | sudo yum reinstall kernel || true
2023-03-25T21:37:06.884 INFO:teuthology.orchestra.run.smithi086.stdout:Updating Subscription Management repositories.
2023-03-25T21:37:06.884 INFO:teuthology.orchestra.run.smithi086.stdout:Unable to read consumer identity
2023-03-25T21:37:06.908 INFO:teuthology.orchestra.run.smithi062.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed.
2023-03-25T21:37:06.953 INFO:teuthology.orchestra.run.smithi062.stdout:Dependencies resolved.
2023-03-25T21:37:06.954 INFO:teuthology.orchestra.run.smithi062.stdout:Nothing to do.
2023-03-25T21:37:06.954 INFO:teuthology.orchestra.run.smithi062.stdout:Complete!
2023-03-25T21:37:07.067 DEBUG:teuthology.orchestra.run.smithi062:> echo no | sudo yum reinstall kernel || true
2023-03-25T21:37:07.140 INFO:teuthology.orchestra.run.smithi086.stdout:Last metadata expiration check: 0:00:02 ago on Sat 25 Mar 2023 09:37:05 PM UTC.
2023-03-25T21:37:07.181 INFO:teuthology.orchestra.run.smithi086.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available.
2023-03-25T21:37:07.222 INFO:teuthology.orchestra.run.smithi086.stderr:Error: No packages marked for reinstall.
2023-03-25T21:37:07.267 DEBUG:teuthology.orchestra.run.smithi086:> sudo yum reinstall -y kernel || true
2023-03-25T21:37:07.457 INFO:teuthology.orchestra.run.smithi062.stdout:Updating Subscription Management repositories.
2023-03-25T21:37:07.458 INFO:teuthology.orchestra.run.smithi062.stdout:Unable to read consumer identity
2023-03-25T21:37:07.594 INFO:teuthology.orchestra.run.smithi086.stdout:Updating Subscription Management repositories.
2023-03-25T21:37:07.595 INFO:teuthology.orchestra.run.smithi086.stdout:Unable to read consumer identity
2023-03-25T21:37:07.726 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:00:01 ago on Sat 25 Mar 2023 09:37:06 PM UTC.
2023-03-25T21:37:07.771 INFO:teuthology.orchestra.run.smithi062.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available.
2023-03-25T21:37:07.803 INFO:teuthology.orchestra.run.smithi062.stderr:Error: No packages marked for reinstall.
2023-03-25T21:37:07.844 INFO:teuthology.orchestra.run.smithi086.stdout:Last metadata expiration check: 0:00:02 ago on Sat 25 Mar 2023 09:37:05 PM UTC.
2023-03-25T21:37:07.851 DEBUG:teuthology.orchestra.run.smithi062:> sudo yum reinstall -y kernel || true
2023-03-25T21:37:07.884 INFO:teuthology.orchestra.run.smithi086.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available.
2023-03-25T21:37:07.930 INFO:teuthology.orchestra.run.smithi086.stderr:Error: No packages marked for reinstall.
2023-03-25T21:37:07.975 DEBUG:teuthology.orchestra.run.smithi086:> rpm -q kernel --last | head -n 1
2023-03-25T21:37:08.054 INFO:teuthology.orchestra.run.smithi086.stdout:kernel-4.18.0-372.9.1.el8.x86_64              Thu 14 Jul 2022 06:23:21 PM UTC
2023-03-25T21:37:08.055 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64
2023-03-25T21:37:08.056 INFO:teuthology.task.kernel:Newest distro kernel installed and running
2023-03-25T21:37:08.056 INFO:teuthology.task.kernel:Enabling kdb on mon.b...
2023-03-25T21:37:08.056 DEBUG:teuthology.orchestra.run.smithi086:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2023-03-25T21:37:08.088 INFO:teuthology.orchestra.run.smithi086.stdout:ttyS1
2023-03-25T21:37:08.102 DEBUG:teuthology.parallel:result is None
2023-03-25T21:37:08.200 INFO:teuthology.orchestra.run.smithi062.stdout:Updating Subscription Management repositories.
2023-03-25T21:37:08.201 INFO:teuthology.orchestra.run.smithi062.stdout:Unable to read consumer identity
2023-03-25T21:37:08.466 INFO:teuthology.orchestra.run.smithi062.stdout:Last metadata expiration check: 0:00:02 ago on Sat 25 Mar 2023 09:37:06 PM UTC.
2023-03-25T21:37:08.508 INFO:teuthology.orchestra.run.smithi062.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available.
2023-03-25T21:37:08.528 INFO:teuthology.orchestra.run.smithi062.stderr:Error: No packages marked for reinstall.
2023-03-25T21:37:08.576 DEBUG:teuthology.orchestra.run.smithi062:> rpm -q kernel --last | head -n 1
2023-03-25T21:37:08.658 INFO:teuthology.orchestra.run.smithi062.stdout:kernel-4.18.0-372.9.1.el8.x86_64              Thu 14 Jul 2022 06:23:21 PM UTC
2023-03-25T21:37:08.660 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64
2023-03-25T21:37:08.660 INFO:teuthology.task.kernel:Newest distro kernel installed and running
2023-03-25T21:37:08.660 INFO:teuthology.task.kernel:Enabling kdb on mon.a...
2023-03-25T21:37:08.661 DEBUG:teuthology.orchestra.run.smithi062:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2023-03-25T21:37:08.700 INFO:teuthology.orchestra.run.smithi062.stdout:ttyS1
2023-03-25T21:37:08.716 DEBUG:teuthology.parallel:result is None
2023-03-25T21:37:08.716 INFO:teuthology.run_tasks:Running task internal.base...
2023-03-25T21:37:08.725 INFO:teuthology.task.internal:Creating test directory...
2023-03-25T21:37:08.726 DEBUG:teuthology.orchestra.run.smithi062:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2023-03-25T21:37:08.757 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2023-03-25T21:37:08.777 INFO:teuthology.run_tasks:Running task internal.archive_upload...
2023-03-25T21:37:08.788 INFO:teuthology.run_tasks:Running task internal.archive...
2023-03-25T21:37:08.896 INFO:teuthology.task.internal:Creating archive directory...
2023-03-25T21:37:08.896 DEBUG:teuthology.orchestra.run.smithi062:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2023-03-25T21:37:08.898 DEBUG:teuthology.orchestra.run.smithi086:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2023-03-25T21:37:08.933 INFO:teuthology.run_tasks:Running task internal.coredump...
2023-03-25T21:37:08.952 INFO:teuthology.task.internal:Enabling coredump saving...
2023-03-25T21:37:08.952 DEBUG:teuthology.orchestra.run.smithi062:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2023-03-25T21:37:08.964 DEBUG:teuthology.orchestra.run.smithi086:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2023-03-25T21:37:09.001 INFO:teuthology.orchestra.run.smithi062.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2023-03-25T21:37:09.010 INFO:teuthology.orchestra.run.smithi086.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2023-03-25T21:37:09.011 INFO:teuthology.orchestra.run.smithi062.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2023-03-25T21:37:09.021 INFO:teuthology.orchestra.run.smithi086.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2023-03-25T21:37:09.023 INFO:teuthology.run_tasks:Running task internal.sudo...
2023-03-25T21:37:09.034 INFO:teuthology.task.internal:Configuring sudo...
2023-03-25T21:37:09.034 DEBUG:teuthology.orchestra.run.smithi062:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2023-03-25T21:37:09.055 DEBUG:teuthology.orchestra.run.smithi086:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2023-03-25T21:37:09.091 INFO:teuthology.run_tasks:Running task internal.syslog...
2023-03-25T21:37:09.105 INFO:teuthology.task.internal.syslog:Starting syslog monitoring...
2023-03-25T21:37:09.106 DEBUG:teuthology.orchestra.run.smithi062:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2023-03-25T21:37:09.121 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2023-03-25T21:37:09.149 DEBUG:teuthology.orchestra.run.smithi062:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2023-03-25T21:37:09.222 DEBUG:teuthology.orchestra.run.smithi062:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log
2023-03-25T21:37:09.280 DEBUG:teuthology.orchestra.run.smithi062:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2023-03-25T21:37:09.324 DEBUG:teuthology.orchestra.run.smithi062:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log
2023-03-25T21:37:09.365 DEBUG:teuthology.orchestra.run.smithi062:> set -ex
2023-03-25T21:37:09.365 DEBUG:teuthology.orchestra.run.smithi062:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2023-03-25T21:37:09.432 DEBUG:teuthology.orchestra.run.smithi086:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2023-03-25T21:37:09.481 DEBUG:teuthology.orchestra.run.smithi086:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log
2023-03-25T21:37:09.528 DEBUG:teuthology.orchestra.run.smithi086:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2023-03-25T21:37:09.613 DEBUG:teuthology.orchestra.run.smithi086:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log
2023-03-25T21:37:09.656 DEBUG:teuthology.orchestra.run.smithi086:> set -ex
2023-03-25T21:37:09.656 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2023-03-25T21:37:09.724 DEBUG:teuthology.orchestra.run.smithi062:> sudo service rsyslog restart
2023-03-25T21:37:09.726 DEBUG:teuthology.orchestra.run.smithi086:> sudo service rsyslog restart
2023-03-25T21:37:09.782 INFO:teuthology.orchestra.run.smithi062.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2023-03-25T21:37:09.829 INFO:teuthology.orchestra.run.smithi086.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2023-03-25T21:37:10.214 INFO:teuthology.run_tasks:Running task internal.timer...
2023-03-25T21:37:10.223 INFO:teuthology.task.internal:Starting timer...
2023-03-25T21:37:10.224 INFO:teuthology.run_tasks:Running task pcp...
2023-03-25T21:37:10.261 INFO:teuthology.run_tasks:Running task selinux...
2023-03-25T21:37:10.287 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}
2023-03-25T21:37:10.288 DEBUG:teuthology.orchestra.run.smithi062:> sudo service auditd rotate
2023-03-25T21:37:10.355 INFO:teuthology.orchestra.run.smithi062.stdout:Rotating logs:
2023-03-25T21:37:10.356 DEBUG:teuthology.orchestra.run.smithi086:> sudo service auditd rotate
2023-03-25T21:37:10.408 INFO:teuthology.orchestra.run.smithi086.stdout:Rotating logs:
2023-03-25T21:37:10.409 DEBUG:teuthology.task.selinux:Getting current SELinux state
2023-03-25T21:37:10.410 DEBUG:teuthology.orchestra.run.smithi062:> /usr/sbin/getenforce
2023-03-25T21:37:10.434 INFO:teuthology.orchestra.run.smithi062.stdout:Permissive
2023-03-25T21:37:10.434 DEBUG:teuthology.orchestra.run.smithi086:> /usr/sbin/getenforce
2023-03-25T21:37:10.457 INFO:teuthology.orchestra.run.smithi086.stdout:Permissive
2023-03-25T21:37:10.458 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi062.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi086.front.sepia.ceph.com': 'permissive'}
2023-03-25T21:37:10.458 DEBUG:teuthology.orchestra.run.smithi062:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2023-03-25T21:37:10.501 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-03-25T21:37:10.502 DEBUG:teuthology.orchestra.run.smithi086:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2023-03-25T21:37:10.530 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-03-25T21:37:10.531 INFO:teuthology.task.selinux:Putting SELinux into permissive mode
2023-03-25T21:37:10.532 DEBUG:teuthology.orchestra.run.smithi062:> sudo /usr/sbin/setenforce permissive
2023-03-25T21:37:10.571 DEBUG:teuthology.orchestra.run.smithi086:> sudo /usr/sbin/setenforce permissive
2023-03-25T21:37:10.600 INFO:teuthology.run_tasks:Running task ansible.cephlab...
2023-03-25T21:37:10.612 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current
2023-03-25T21:37:10.613 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main
2023-03-25T21:37:10.638 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}]
2023-03-25T21:37:10.640 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi062.front.sepia.ceph.com,smithi086.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml
2023-03-25T21:41:14.198 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/lflores-2023-03-25_21:26:30-rados-wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a-distro-default-smithi/7219689/ansible_failures.yaml
2023-03-25T21:41:14.200 ERROR:teuthology.run_tasks:Saw exception from tasks.
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 109, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/ansible.py", line 426, in begin
    super(CephLab, self).begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/ansible.py", line 267, in begin
    self.execute_playbook()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/ansible.py", line 295, in execute_playbook
    self._handle_failure(command, status)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/ansible.py", line 320, in _handle_failure
    raise AnsibleFailedError(failures)
teuthology.exceptions.AnsibleFailedError: {'smithi086.front.sepia.ceph.com': {'_ansible_no_log': False, 'msg': "Failed to connect to the host via ssh: Warning: Permanently added 'smithi086.front.sepia.ceph.com,172.21.15.86' (ECDSA) to the list of known hosts.\r\nubuntu@smithi086.front.sepia.ceph.com: Permission denied (publickey,password,keyboard-interactive)."}, 'smithi062.front.sepia.ceph.com': {'_ansible_no_log': False, 'msg': "Failed to connect to the host via ssh: Warning: Permanently added 'smithi062.front.sepia.ceph.com,172.21.15.62' (ECDSA) to the list of known hosts.\r\nubuntu@smithi062.front.sepia.ceph.com: Permission denied (publickey,password,keyboard-interactive)."}}
2023-03-25T21:41:14.286 ERROR:teuthology.run_tasks: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=50bfa58268c344469b3b3c3a1d928793
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 109, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/ansible.py", line 426, in begin
    super(CephLab, self).begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/ansible.py", line 267, in begin
    self.execute_playbook()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/ansible.py", line 295, in execute_playbook
    self._handle_failure(command, status)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/ansible.py", line 320, in _handle_failure
    raise AnsibleFailedError(failures)
teuthology.exceptions.AnsibleFailedError: {'smithi086.front.sepia.ceph.com': {'_ansible_no_log': False, 'msg': "Failed to connect to the host via ssh: Warning: Permanently added 'smithi086.front.sepia.ceph.com,172.21.15.86' (ECDSA) to the list of known hosts.\r\nubuntu@smithi086.front.sepia.ceph.com: Permission denied (publickey,password,keyboard-interactive)."}, 'smithi062.front.sepia.ceph.com': {'_ansible_no_log': False, 'msg': "Failed to connect to the host via ssh: Warning: Permanently added 'smithi062.front.sepia.ceph.com,172.21.15.62' (ECDSA) to the list of known hosts.\r\nubuntu@smithi062.front.sepia.ceph.com: Permission denied (publickey,password,keyboard-interactive)."}}
2023-03-25T21:41:14.289 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab
2023-03-25T21:41:14.297 INFO:teuthology.task.ansible:Skipping ansible cleanup...
2023-03-25T21:41:14.298 DEBUG:teuthology.run_tasks:Unwinding manager selinux
2023-03-25T21:41:14.332 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi062.front.sepia.ceph.com'
2023-03-25T21:41:14.333 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi062.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-03-25T21:41:14.424 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:15.544 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:16.666 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:17.786 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:18.906 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:20.026 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:21.107 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:22.231 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:23.351 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:24.471 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:24.474 DEBUG:teuthology.orchestra.remote:'connect to smithi062.front.sepia.ceph.com' reached maximum tries (10) after waiting for 10 seconds
2023-03-25T21:41:24.475 ERROR:teuthology.run_tasks:Manager failed: selinux
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 192, in run_tasks
    suppress = manager.__exit__(*exc_info)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/__init__.py", line 136, in __exit__
    self.teardown()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/selinux.py", line 165, in teardown
    self.archive_log()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/selinux.py", line 195, in archive_log
    self.cluster.run(
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in run
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in <listcomp>
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 524, in run
    raise Exception(f'Cannot connect to remote host {self.shortname}')
Exception: Cannot connect to remote host smithi062
2023-03-25T21:41:24.481 DEBUG:teuthology.run_tasks:Unwinding manager pcp
2023-03-25T21:41:24.490 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer
2023-03-25T21:41:24.517 INFO:teuthology.task.internal:Duration was 254.293531 seconds
2023-03-25T21:41:24.518 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog
2023-03-25T21:41:24.558 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring...
2023-03-25T21:41:24.559 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi062.front.sepia.ceph.com'
2023-03-25T21:41:24.559 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi062.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-03-25T21:41:24.677 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:25.800 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:26.925 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:28.045 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:29.167 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:30.289 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:31.295 ERROR:paramiko.transport:Exception (client): Error reading SSH protocol banner
2023-03-25T21:41:31.324 ERROR:paramiko.transport:Traceback (most recent call last):
2023-03-25T21:41:31.328 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2270, in _check_banner
2023-03-25T21:41:31.331 ERROR:paramiko.transport:    buf = self.packetizer.readline(timeout)
2023-03-25T21:41:31.334 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/packet.py", line 380, in readline
2023-03-25T21:41:31.337 ERROR:paramiko.transport:    buf += self._read_timeout(timeout)
2023-03-25T21:41:31.340 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/packet.py", line 609, in _read_timeout
2023-03-25T21:41:31.343 ERROR:paramiko.transport:    raise EOFError()
2023-03-25T21:41:31.346 ERROR:paramiko.transport:EOFError
2023-03-25T21:41:31.349 ERROR:paramiko.transport:
2023-03-25T21:41:31.352 ERROR:paramiko.transport:During handling of the above exception, another exception occurred:
2023-03-25T21:41:31.355 ERROR:paramiko.transport:
2023-03-25T21:41:31.358 ERROR:paramiko.transport:Traceback (most recent call last):
2023-03-25T21:41:31.361 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2093, in run
2023-03-25T21:41:31.364 ERROR:paramiko.transport:    self._check_banner()
2023-03-25T21:41:31.368 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2274, in _check_banner
2023-03-25T21:41:31.371 ERROR:paramiko.transport:    raise SSHException(
2023-03-25T21:41:31.374 ERROR:paramiko.transport:paramiko.ssh_exception.SSHException: Error reading SSH protocol banner
2023-03-25T21:41:31.377 ERROR:paramiko.transport:
2023-03-25T21:41:31.381 DEBUG:teuthology.orchestra.remote:Error reading SSH protocol banner
2023-03-25T21:41:31.382 ERROR:teuthology.run_tasks:Manager failed: internal.syslog
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/syslog.py", line 76, in syslog
    yield
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 192, in run_tasks
    suppress = manager.__exit__(*exc_info)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/__init__.py", line 136, in __exit__
    self.teardown()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/selinux.py", line 165, in teardown
    self.archive_log()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/selinux.py", line 195, in archive_log
    self.cluster.run(
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in run
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in <listcomp>
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 524, in run
    raise Exception(f'Cannot connect to remote host {self.shortname}')
Exception: Cannot connect to remote host smithi062

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 192, in run_tasks
    suppress = manager.__exit__(*exc_info)
  File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__
    self.gen.throw(type, value, traceback)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/syslog.py", line 85, in syslog
    cluster.run(
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in run
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in <listcomp>
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 524, in run
    raise Exception(f'Cannot connect to remote host {self.shortname}')
Exception: Cannot connect to remote host smithi062
2023-03-25T21:41:31.390 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo
2023-03-25T21:41:31.399 INFO:teuthology.task.internal:Restoring /etc/sudoers...
2023-03-25T21:41:31.400 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi062.front.sepia.ceph.com'
2023-03-25T21:41:31.401 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi062.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-03-25T21:41:31.519 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:32.640 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:33.760 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:34.766 ERROR:paramiko.transport:Exception (client): Error reading SSH protocol banner
2023-03-25T21:41:34.770 ERROR:paramiko.transport:Traceback (most recent call last):
2023-03-25T21:41:34.774 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2270, in _check_banner
2023-03-25T21:41:34.777 ERROR:paramiko.transport:    buf = self.packetizer.readline(timeout)
2023-03-25T21:41:34.780 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/packet.py", line 380, in readline
2023-03-25T21:41:34.783 ERROR:paramiko.transport:    buf += self._read_timeout(timeout)
2023-03-25T21:41:34.786 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/packet.py", line 609, in _read_timeout
2023-03-25T21:41:34.789 ERROR:paramiko.transport:    raise EOFError()
2023-03-25T21:41:34.792 ERROR:paramiko.transport:EOFError
2023-03-25T21:41:34.795 ERROR:paramiko.transport:
2023-03-25T21:41:34.799 ERROR:paramiko.transport:During handling of the above exception, another exception occurred:
2023-03-25T21:41:34.802 ERROR:paramiko.transport:
2023-03-25T21:41:34.805 ERROR:paramiko.transport:Traceback (most recent call last):
2023-03-25T21:41:34.808 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2093, in run
2023-03-25T21:41:34.812 ERROR:paramiko.transport:    self._check_banner()
2023-03-25T21:41:34.815 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2274, in _check_banner
2023-03-25T21:41:34.818 ERROR:paramiko.transport:    raise SSHException(
2023-03-25T21:41:34.822 ERROR:paramiko.transport:paramiko.ssh_exception.SSHException: Error reading SSH protocol banner
2023-03-25T21:41:34.825 ERROR:paramiko.transport:
2023-03-25T21:41:34.829 DEBUG:teuthology.orchestra.remote:Error reading SSH protocol banner
2023-03-25T21:41:34.830 ERROR:teuthology.run_tasks:Manager failed: internal.sudo
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/__init__.py", line 452, in sudo
    yield
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 192, in run_tasks
    suppress = manager.__exit__(*exc_info)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/__init__.py", line 136, in __exit__
    self.teardown()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/selinux.py", line 165, in teardown
    self.archive_log()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/selinux.py", line 195, in archive_log
    self.cluster.run(
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in run
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in <listcomp>
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 524, in run
    raise Exception(f'Cannot connect to remote host {self.shortname}')
Exception: Cannot connect to remote host smithi062

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 192, in run_tasks
    suppress = manager.__exit__(*exc_info)
  File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__
    self.gen.throw(type, value, traceback)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/__init__.py", line 455, in sudo
    ctx.cluster.run(
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in run
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in <listcomp>
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 524, in run
    raise Exception(f'Cannot connect to remote host {self.shortname}')
Exception: Cannot connect to remote host smithi062
2023-03-25T21:41:34.837 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
2023-03-25T21:41:34.847 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi062.front.sepia.ceph.com'
2023-03-25T21:41:34.848 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi062.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-03-25T21:41:34.850 ERROR:paramiko.transport:Exception (client): Error reading SSH protocol banner
2023-03-25T21:41:34.853 ERROR:paramiko.transport:Traceback (most recent call last):
2023-03-25T21:41:34.856 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2270, in _check_banner
2023-03-25T21:41:34.859 ERROR:paramiko.transport:    buf = self.packetizer.readline(timeout)
2023-03-25T21:41:34.863 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/packet.py", line 380, in readline
2023-03-25T21:41:34.866 ERROR:paramiko.transport:    buf += self._read_timeout(timeout)
2023-03-25T21:41:34.869 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/packet.py", line 609, in _read_timeout
2023-03-25T21:41:34.872 ERROR:paramiko.transport:    raise EOFError()
2023-03-25T21:41:34.875 ERROR:paramiko.transport:EOFError
2023-03-25T21:41:34.879 ERROR:paramiko.transport:
2023-03-25T21:41:34.882 ERROR:paramiko.transport:During handling of the above exception, another exception occurred:
2023-03-25T21:41:34.885 ERROR:paramiko.transport:
2023-03-25T21:41:34.889 ERROR:paramiko.transport:Traceback (most recent call last):
2023-03-25T21:41:34.892 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2093, in run
2023-03-25T21:41:34.895 ERROR:paramiko.transport:    self._check_banner()
2023-03-25T21:41:34.899 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2274, in _check_banner
2023-03-25T21:41:34.902 ERROR:paramiko.transport:    raise SSHException(
2023-03-25T21:41:34.906 ERROR:paramiko.transport:paramiko.ssh_exception.SSHException: Error reading SSH protocol banner
2023-03-25T21:41:34.910 ERROR:paramiko.transport:
2023-03-25T21:41:34.913 DEBUG:teuthology.orchestra.remote:Error reading SSH protocol banner
2023-03-25T21:41:34.914 ERROR:teuthology.run_tasks:Manager failed: internal.coredump
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/__init__.py", line 488, in coredump
    yield
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 192, in run_tasks
    suppress = manager.__exit__(*exc_info)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/__init__.py", line 136, in __exit__
    self.teardown()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/selinux.py", line 165, in teardown
    self.archive_log()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/selinux.py", line 195, in archive_log
    self.cluster.run(
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in run
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in <listcomp>
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 524, in run
    raise Exception(f'Cannot connect to remote host {self.shortname}')
Exception: Cannot connect to remote host smithi062

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 192, in run_tasks
    suppress = manager.__exit__(*exc_info)
  File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__
    self.gen.throw(type, value, traceback)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/__init__.py", line 492, in coredump
    cluster.run(
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in run
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in <listcomp>
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 524, in run
    raise Exception(f'Cannot connect to remote host {self.shortname}')
Exception: Cannot connect to remote host smithi062
2023-03-25T21:41:34.921 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive
2023-03-25T21:41:34.930 INFO:teuthology.task.internal:Transferring archived files...
2023-03-25T21:41:34.932 DEBUG:teuthology.misc:Transferring archived files from smithi062:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/lflores-2023-03-25_21:26:30-rados-wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a-distro-default-smithi/7219689/remote/smithi062
2023-03-25T21:41:34.933 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi062.front.sepia.ceph.com'
2023-03-25T21:41:34.934 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi062.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-03-25T21:41:34.938 ERROR:paramiko.transport:Exception (client): Error reading SSH protocol banner
2023-03-25T21:41:34.941 ERROR:paramiko.transport:Traceback (most recent call last):
2023-03-25T21:41:34.944 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2270, in _check_banner
2023-03-25T21:41:34.947 ERROR:paramiko.transport:    buf = self.packetizer.readline(timeout)
2023-03-25T21:41:34.950 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/packet.py", line 380, in readline
2023-03-25T21:41:34.954 ERROR:paramiko.transport:    buf += self._read_timeout(timeout)
2023-03-25T21:41:34.957 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/packet.py", line 609, in _read_timeout
2023-03-25T21:41:34.960 ERROR:paramiko.transport:    raise EOFError()
2023-03-25T21:41:34.964 ERROR:paramiko.transport:EOFError
2023-03-25T21:41:34.967 ERROR:paramiko.transport:
2023-03-25T21:41:34.970 ERROR:paramiko.transport:During handling of the above exception, another exception occurred:
2023-03-25T21:41:34.974 ERROR:paramiko.transport:
2023-03-25T21:41:34.977 ERROR:paramiko.transport:Traceback (most recent call last):
2023-03-25T21:41:34.981 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2093, in run
2023-03-25T21:41:34.985 ERROR:paramiko.transport:    self._check_banner()
2023-03-25T21:41:34.988 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2274, in _check_banner
2023-03-25T21:41:34.992 ERROR:paramiko.transport:    raise SSHException(
2023-03-25T21:41:34.995 ERROR:paramiko.transport:paramiko.ssh_exception.SSHException: Error reading SSH protocol banner
2023-03-25T21:41:34.999 ERROR:paramiko.transport:
2023-03-25T21:41:35.003 DEBUG:teuthology.orchestra.remote:Error reading SSH protocol banner
2023-03-25T21:41:35.004 ERROR:teuthology.run_tasks:Manager failed: internal.archive
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/__init__.py", line 392, in archive
    yield
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 192, in run_tasks
    suppress = manager.__exit__(*exc_info)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/__init__.py", line 136, in __exit__
    self.teardown()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/selinux.py", line 165, in teardown
    self.archive_log()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/selinux.py", line 195, in archive_log
    self.cluster.run(
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in run
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in <listcomp>
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 524, in run
    raise Exception(f'Cannot connect to remote host {self.shortname}')
Exception: Cannot connect to remote host smithi062

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 192, in run_tasks
    suppress = manager.__exit__(*exc_info)
  File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__
    self.gen.throw(type, value, traceback)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/__init__.py", line 418, in archive
    misc.pull_directory(rem, archive_dir, path, maybe_compress)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/misc.py", line 748, in pull_directory
    r = remote.get_tar_stream(remotedir, sudo=True)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 664, in get_tar_stream
    return self.run(args=args, wait=False, stdout=run.PIPE)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 524, in run
    raise Exception(f'Cannot connect to remote host {self.shortname}')
Exception: Cannot connect to remote host smithi062
2023-03-25T21:41:35.011 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload
2023-03-25T21:41:35.020 INFO:teuthology.task.internal:Not uploading archives.
2023-03-25T21:41:35.021 DEBUG:teuthology.run_tasks:Unwinding manager internal.base
2023-03-25T21:41:35.050 INFO:teuthology.task.internal:Tidying up after the test...
2023-03-25T21:41:35.051 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi062.front.sepia.ceph.com'
2023-03-25T21:41:35.051 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi062.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-03-25T21:41:35.171 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:36.294 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:37.417 ERROR:teuthology.orchestra.connection:Error authenticating with smithi062.front.sepia.ceph.com: Authentication failed.
2023-03-25T21:41:38.424 ERROR:paramiko.transport:Exception (client): Error reading SSH protocol banner
2023-03-25T21:41:38.429 ERROR:paramiko.transport:Traceback (most recent call last):
2023-03-25T21:41:38.435 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2270, in _check_banner
2023-03-25T21:41:38.439 ERROR:paramiko.transport:    buf = self.packetizer.readline(timeout)
2023-03-25T21:41:38.442 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/packet.py", line 380, in readline
2023-03-25T21:41:38.446 ERROR:paramiko.transport:    buf += self._read_timeout(timeout)
2023-03-25T21:41:38.450 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/packet.py", line 609, in _read_timeout
2023-03-25T21:41:38.454 ERROR:paramiko.transport:    raise EOFError()
2023-03-25T21:41:38.458 ERROR:paramiko.transport:EOFError
2023-03-25T21:41:38.462 ERROR:paramiko.transport:
2023-03-25T21:41:38.466 ERROR:paramiko.transport:During handling of the above exception, another exception occurred:
2023-03-25T21:41:38.470 ERROR:paramiko.transport:
2023-03-25T21:41:38.474 ERROR:paramiko.transport:Traceback (most recent call last):
2023-03-25T21:41:38.478 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2093, in run
2023-03-25T21:41:38.482 ERROR:paramiko.transport:    self._check_banner()
2023-03-25T21:41:38.486 ERROR:paramiko.transport:  File "/home/teuthworker/src/git.ceph.com_teuthology_main/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 2274, in _check_banner
2023-03-25T21:41:38.490 ERROR:paramiko.transport:    raise SSHException(
2023-03-25T21:41:38.494 ERROR:paramiko.transport:paramiko.ssh_exception.SSHException: Error reading SSH protocol banner
2023-03-25T21:41:38.499 ERROR:paramiko.transport:
2023-03-25T21:41:38.503 DEBUG:teuthology.orchestra.remote:Error reading SSH protocol banner
2023-03-25T21:41:38.504 ERROR:teuthology.run_tasks:Manager failed: internal.base
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/__init__.py", line 48, in base
    yield
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 192, in run_tasks
    suppress = manager.__exit__(*exc_info)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/__init__.py", line 136, in __exit__
    self.teardown()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/selinux.py", line 165, in teardown
    self.archive_log()
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/selinux.py", line 195, in archive_log
    self.cluster.run(
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in run
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in <listcomp>
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 524, in run
    raise Exception(f'Cannot connect to remote host {self.shortname}')
Exception: Cannot connect to remote host smithi062

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 192, in run_tasks
    suppress = manager.__exit__(*exc_info)
  File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__
    self.gen.throw(type, value, traceback)
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/__init__.py", line 54, in base
    ctx.cluster.run(
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in run
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/cluster.py", line 85, in <listcomp>
    procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 524, in run
    raise Exception(f'Cannot connect to remote host {self.shortname}')
Exception: Cannot connect to remote host smithi062
2023-03-25T21:41:38.512 DEBUG:teuthology.run_tasks:Unwinding manager kernel
2023-03-25T21:41:38.522 DEBUG:teuthology.run_tasks:Unwinding manager console_log
2023-03-25T21:41:38.588 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: Exception: Cannot connect to remote host smithi062
2023-03-25T21:41:38.588 INFO:teuthology.nuke:Checking targets against current locks
2023-03-25T21:41:38.622 INFO:teuthology.nuke:Not nuking smithi062.front.sepia.ceph.com because description doesn't match: lflores-2023-03-25_21:26:30-rados-wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a-distro-default-smithi != /home/teuthworker/archive/lflores-2023-03-25_21:36:10-rados-wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a-distro-default-smithi/7219773
2023-03-25T21:41:38.651 INFO:teuthology.nuke:Not nuking smithi086.front.sepia.ceph.com because description doesn't match: lflores-2023-03-25_21:26:30-rados-wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a-distro-default-smithi != /home/teuthworker/archive/lflores-2023-03-25_21:36:10-rados-wip-aclamk-bs-elastic-shared-blob-save-25.03.2023-a-distro-default-smithi/7219774
2023-03-25T21:41:38.653 ERROR:teuthology.nuke:Could not nuke the following targets:
targets:
  smithi062.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac=
  smithi086.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac=
2023-03-25T21:41:38.660 INFO:teuthology.run:Summary data:
description: rados/thrash-erasure-code-overwrites/{bluestore-bitmap ceph clusters/{fixed-2
  openstack} fast/fast mon_election/classic msgr-failures/fastclose rados recovery-overrides/{more-active-recovery}
  supported-random-distro$/{rhel_8} thrashers/fastread thrashosds-health workloads/ec-snaps-few-objects-overwrites}
duration: 254.29353094100952
failure_reason: '{''smithi086.front.sepia.ceph.com'': {''_ansible_no_log'': False,
  ''msg'': "Failed to connect to the host via ssh: Warning: Permanently added ''smithi086.front.sepia.ceph.com,172.21.15.86''
  (ECDSA) to the list of known hosts.\r\nubuntu@smithi086.front.sepia.ceph.com: Permission
  denied (publickey,password,keyboard-interactive)."}, ''smithi062.front.sepia.ceph.com'':
  {''_ansible_no_log'': False, ''msg'': "Failed to connect to the host via ssh: Warning:
  Permanently added ''smithi062.front.sepia.ceph.com,172.21.15.62'' (ECDSA) to the
  list of known hosts.\r\nubuntu@smithi062.front.sepia.ceph.com: Permission denied
  (publickey,password,keyboard-interactive)."}}'
owner: scheduled_lflores@teuthology
sentry_event: https://sentry.ceph.com/organizations/ceph/?query=50bfa58268c344469b3b3c3a1d928793
status: fail
success: false

2023-03-25T21:41:38.661 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/
2023-03-25T21:41:38.742 INFO:teuthology.run:FAIL