2023-12-26T18:10:17.351 INFO:root:teuthology version: 0.0.1.dev226+g9680d2a
2023-12-26T18:10:17.352 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi/7501638 --verbose --description rbd/migration/{1-base/install 2-clusters/{fixed-3 openstack} 3-objectstore/bluestore-hybrid 4-supported-random-distro$/{rhel_8} 5-pool/ec-data-pool 6-prepare/qcow2-file 7-io-workloads/qemu_xfstests 8-migrate-workloads/execute 9-cleanup/cleanup} --name yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi --owner scheduled_yuriw@teuthology -- /home/teuthworker/archive/yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi/7501638/orig.config.yaml
2023-12-26T18:10:17.393 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/
2023-12-26T18:10:17.477 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi/7501638
  branch: wip-yuri7-testing-2023-12-20-0808-pacific
  description: rbd/migration/{1-base/install 2-clusters/{fixed-3 openstack} 3-objectstore/bluestore-hybrid
    4-supported-random-distro$/{rhel_8} 5-pool/ec-data-pool 6-prepare/qcow2-file 7-io-workloads/qemu_xfstests
    8-migrate-workloads/execute 9-cleanup/cleanup}
  email: null
  exclude_arch: armv7l
  first_in_suite: false
  io_workload:
    sequential:
    - qemu:
        client.0:
          clone: true
          disks:
          - action: none
            image_name: client.0.0
          - action: none
            image_name: client.0.1
          - action: none
            image_name: client.0.2
          test: qa/run_xfstests_qemu.sh
          type: block
  job_id: '7501638'
  kernel:
    kdb: true
    sha1: distro
  last_in_suite: false
  machine_type: smithi
  migrate_workload:
    sequential:
    - exec:
        client.0:
        - sleep 60
        - rbd migration execute client.0.0
        - sleep 60
        - rbd migration commit client.0.0
        - sleep 60
        - rbd migration execute client.0.1
  name: yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi
  no_nested_subset: false
  nuke-on-error: true
  openstack:
  - volumes:
      count: 4
      size: 10
  - machine:
      cpus: 1
      disk: 40
      ram: 30000
    volumes:
      count: 4
      size: 30
  os_type: rhel
  os_version: '8.4'
  overrides:
    admin_socket:
      branch: wip-yuri7-testing-2023-12-20-0808-pacific
    ceph:
      conf:
        client:
          rbd default data pool: datapool
        mgr:
          debug mgr: 20
          debug ms: 1
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
        osd:
          bluefs allocator: hybrid
          bluestore allocator: hybrid
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          debug bluefs: 1/20
          debug bluestore: 1/20
          debug ms: 1
          debug osd: 20
          debug rocksdb: 4/10
          enable experimental unrecoverable data corrupting features: '*'
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd debug randomize hobject sort order: false
          osd failsafe full ratio: 0.95
          osd objectstore: bluestore
          osd shutdown pgref assert: true
      flavor: default
      fs: xfs
      log-ignorelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      sha1: 639f0137318def3e5f393094e5494880043a8f32
    ceph-deploy:
      bluestore: true
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon: {}
        osd:
          bluestore block size: 96636764160
          bluestore fsck on mount: true
          debug bluefs: 1/20
          debug bluestore: 1/20
          debug rocksdb: 4/10
          mon osd backfillfull_ratio: 0.85
          mon osd full ratio: 0.9
          mon osd nearfull ratio: 0.8
          osd failsafe full ratio: 0.95
          osd objectstore: bluestore
      fs: xfs
    install:
      ceph:
        flavor: default
        sha1: 639f0137318def3e5f393094e5494880043a8f32
    selinux:
      whitelist:
      - scontext=system_u:system_r:logrotate_t:s0
    thrashosds:
      bdev_inject_crash: 2
      bdev_inject_crash_probability: 0.5
    workunit:
      branch: wip-yuri7-testing-2023-12-20-0808-pacific
      sha1: 639f0137318def3e5f393094e5494880043a8f32
  owner: scheduled_yuriw@teuthology
  priority: 75
  repo: https://github.com/ceph/ceph-ci.git
  roles:
  - - mon.a
    - mon.c
    - mgr.x
    - osd.0
    - osd.1
    - osd.2
    - osd.3
  - - mon.b
    - mgr.y
    - osd.4
    - osd.5
    - osd.6
    - osd.7
  - - client.0
  seed: 5573
  sha1: 639f0137318def3e5f393094e5494880043a8f32
  sleep_before_teardown: 0
  subset: 111/120000
  suite: rbd
  suite_branch: wip-yuri7-testing-2023-12-20-0808-pacific
  suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_639f0137318def3e5f393094e5494880043a8f32/qa
  suite_relpath: qa
  suite_repo: https://github.com/ceph/ceph-ci.git
  suite_sha1: 639f0137318def3e5f393094e5494880043a8f32
  targets:
    smithi145.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFFIfNkyXHimgqOvMklW1zmsbUFLzqBd2H1r1kjoQpD8oKkpqkvq6N69HShfxzrjRI+gMAA9jdmCuGhNwCqWCzk=
    smithi159.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG8OL7Gf4ULcwMfSrr0LGQ0mNJVy53yuJWqdfMnOeRRej74j7gq1C+dnUdsCEZq/6BOM6rqMbeudFqsWJTcQPX0=
    smithi183.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM2rv4UQndnVu33kHwptvszMFUQBTzzlU3nV8drYSf58Mudkatvf3KSkjsStMdm0tErpbYuR5/u3v9Jvhv7hATQ=
  tasks:
  - install: null
  - ceph: null
  - exec:
      client.0:
      - sudo ceph osd erasure-code-profile set teuthologyprofile crush-failure-domain=osd
        m=1 k=2
      - sudo ceph osd pool create datapool 4 4 erasure teuthologyprofile
      - sudo ceph osd pool set datapool allow_ec_overwrites true
      - rbd pool init datapool
  - exec:
      client.0:
      - mkdir /home/ubuntu/cephtest/migration
      - wget -nv -O /home/ubuntu/cephtest/migration/base.client.0.qcow2 http://download.ceph.com/qa/ubuntu-12.04.qcow2
      - qemu-img create -f qcow2 /home/ubuntu/cephtest/migration/empty.qcow2 1G
      - echo '{"type":"qcow","stream":{"type":"file","file_path":"/home/ubuntu/cephtest/migration/base.client.0.qcow2"}}'
        | rbd migration prepare --import-only --source-spec-path - client.0.0
      - rbd migration prepare --import-only --source-spec '{"type":"qcow","stream":{"type":"file","file_path":"/home/ubuntu/cephtest/migration/empty.qcow2"}}'
        client.0.1
      - rbd migration prepare --import-only --source-spec '{"type":"qcow","stream":{"type":"file","file_path":"/home/ubuntu/cephtest/migration/empty.qcow2"}}'
        client.0.2
  - parallel:
    - io_workload
    - migrate_workload
  - exec:
      client.0:
      - rm -rf /home/ubuntu/cephtest/migration
  teuthology:
    fragments_dropped: []
    meta: {}
    postmerge: []
  teuthology_branch: main
  teuthology_sha1: 9680d2aba1e76f64ade295c81599c97907457848
  timestamp: 2023-12-26_16:45:04
  tube: smithi
  user: yuriw
  verbose: true
  worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.1485078
2023-12-26T18:10:17.477 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_639f0137318def3e5f393094e5494880043a8f32/qa; will attempt to use it
2023-12-26T18:10:17.478 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_639f0137318def3e5f393094e5494880043a8f32/qa/tasks
2023-12-26T18:10:17.479 INFO:teuthology.run_tasks:Running task internal.check_packages...
2023-12-26T18:10:17.481 INFO:teuthology.task.internal:Checking packages...
2023-12-26T18:10:17.505 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash '639f0137318def3e5f393094e5494880043a8f32'
2023-12-26T18:10:17.505 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2023-12-26T18:10:17.505 INFO:teuthology.packaging:ref: None
2023-12-26T18:10:17.505 INFO:teuthology.packaging:tag: None
2023-12-26T18:10:17.505 INFO:teuthology.packaging:branch: wip-yuri7-testing-2023-12-20-0808-pacific
2023-12-26T18:10:17.506 INFO:teuthology.packaging:sha1: 639f0137318def3e5f393094e5494880043a8f32
2023-12-26T18:10:17.506 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=wip-yuri7-testing-2023-12-20-0808-pacific
2023-12-26T18:10:17.650 INFO:teuthology.task.internal:Found packages for ceph version 16.2.14-413.g639f0137
2023-12-26T18:10:17.651 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2023-12-26T18:10:17.657 INFO:teuthology.task.internal:no buildpackages task found
2023-12-26T18:10:17.657 INFO:teuthology.run_tasks:Running task internal.save_config...
2023-12-26T18:10:17.693 INFO:teuthology.task.internal:Saving configuration
2023-12-26T18:10:17.712 INFO:teuthology.run_tasks:Running task internal.check_lock...
2023-12-26T18:10:17.727 INFO:teuthology.task.internal.check_lock:Checking locks...
2023-12-26T18:10:17.753 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi145.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi/7501638', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.4', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-12-26 18:04:59.566298', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFFIfNkyXHimgqOvMklW1zmsbUFLzqBd2H1r1kjoQpD8oKkpqkvq6N69HShfxzrjRI+gMAA9jdmCuGhNwCqWCzk='}
2023-12-26T18:10:17.778 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi159.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi/7501638', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.4', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-12-26 18:04:59.564572', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG8OL7Gf4ULcwMfSrr0LGQ0mNJVy53yuJWqdfMnOeRRej74j7gq1C+dnUdsCEZq/6BOM6rqMbeudFqsWJTcQPX0='}
2023-12-26T18:10:17.800 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi183.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi/7501638', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.4', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-12-26 18:04:59.567816', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM2rv4UQndnVu33kHwptvszMFUQBTzzlU3nV8drYSf58Mudkatvf3KSkjsStMdm0tErpbYuR5/u3v9Jvhv7hATQ='}
2023-12-26T18:10:17.800 INFO:teuthology.run_tasks:Running task internal.add_remotes...
2023-12-26T18:10:17.805 INFO:teuthology.task.internal:roles: ubuntu@smithi145.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.x', 'osd.0', 'osd.1', 'osd.2', 'osd.3']
2023-12-26T18:10:17.806 INFO:teuthology.task.internal:roles: ubuntu@smithi159.front.sepia.ceph.com - ['mon.b', 'mgr.y', 'osd.4', 'osd.5', 'osd.6', 'osd.7']
2023-12-26T18:10:17.806 INFO:teuthology.task.internal:roles: ubuntu@smithi183.front.sepia.ceph.com - ['client.0']
2023-12-26T18:10:17.806 INFO:teuthology.run_tasks:Running task console_log...
2023-12-26T18:10:17.937 DEBUG:teuthology.exit:Installing handler: Handler(exiter=<teuthology.exit.Exiter object at 0x7fac24d35d60>, func=<function ConsoleLog.start_logging.<locals>.kill_console_loggers at 0x7fac26124e50>, signals=[15])
2023-12-26T18:10:17.938 INFO:teuthology.run_tasks:Running task internal.connect...
2023-12-26T18:10:17.947 INFO:teuthology.task.internal:Opening connections...
2023-12-26T18:10:17.947 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi145.front.sepia.ceph.com
2023-12-26T18:10:17.949 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-12-26T18:10:18.016 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi159.front.sepia.ceph.com
2023-12-26T18:10:18.017 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-12-26T18:10:18.085 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi183.front.sepia.ceph.com
2023-12-26T18:10:18.086 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi183.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-12-26T18:10:18.151 INFO:teuthology.run_tasks:Running task internal.push_inventory...
2023-12-26T18:10:18.159 DEBUG:teuthology.orchestra.run.smithi145:> uname -m
2023-12-26T18:10:18.176 INFO:teuthology.orchestra.run.smithi145.stdout:x86_64
2023-12-26T18:10:18.177 DEBUG:teuthology.orchestra.run.smithi145:> cat /etc/os-release
2023-12-26T18:10:18.232 INFO:teuthology.orchestra.run.smithi145.stdout:NAME="Red Hat Enterprise Linux"
2023-12-26T18:10:18.232 INFO:teuthology.orchestra.run.smithi145.stdout:VERSION="8.4 (Ootpa)"
2023-12-26T18:10:18.232 INFO:teuthology.orchestra.run.smithi145.stdout:ID="rhel"
2023-12-26T18:10:18.232 INFO:teuthology.orchestra.run.smithi145.stdout:ID_LIKE="fedora"
2023-12-26T18:10:18.232 INFO:teuthology.orchestra.run.smithi145.stdout:VERSION_ID="8.4"
2023-12-26T18:10:18.232 INFO:teuthology.orchestra.run.smithi145.stdout:PLATFORM_ID="platform:el8"
2023-12-26T18:10:18.232 INFO:teuthology.orchestra.run.smithi145.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.4 (Ootpa)"
2023-12-26T18:10:18.232 INFO:teuthology.orchestra.run.smithi145.stdout:ANSI_COLOR="0;31"
2023-12-26T18:10:18.233 INFO:teuthology.orchestra.run.smithi145.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8.4:GA"
2023-12-26T18:10:18.233 INFO:teuthology.orchestra.run.smithi145.stdout:HOME_URL="https://www.redhat.com/"
2023-12-26T18:10:18.233 INFO:teuthology.orchestra.run.smithi145.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/"
2023-12-26T18:10:18.233 INFO:teuthology.orchestra.run.smithi145.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/"
2023-12-26T18:10:18.233 INFO:teuthology.orchestra.run.smithi145.stdout:
2023-12-26T18:10:18.233 INFO:teuthology.orchestra.run.smithi145.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8"
2023-12-26T18:10:18.233 INFO:teuthology.orchestra.run.smithi145.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.4
2023-12-26T18:10:18.233 INFO:teuthology.orchestra.run.smithi145.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
2023-12-26T18:10:18.233 INFO:teuthology.orchestra.run.smithi145.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.4"
2023-12-26T18:10:18.234 INFO:teuthology.lock.ops:Updating smithi145.front.sepia.ceph.com on lock server
2023-12-26T18:10:18.259 DEBUG:teuthology.orchestra.run.smithi159:> uname -m
2023-12-26T18:10:18.277 INFO:teuthology.orchestra.run.smithi159.stdout:x86_64
2023-12-26T18:10:18.277 DEBUG:teuthology.orchestra.run.smithi159:> cat /etc/os-release
2023-12-26T18:10:18.334 INFO:teuthology.orchestra.run.smithi159.stdout:NAME="Red Hat Enterprise Linux"
2023-12-26T18:10:18.334 INFO:teuthology.orchestra.run.smithi159.stdout:VERSION="8.4 (Ootpa)"
2023-12-26T18:10:18.334 INFO:teuthology.orchestra.run.smithi159.stdout:ID="rhel"
2023-12-26T18:10:18.334 INFO:teuthology.orchestra.run.smithi159.stdout:ID_LIKE="fedora"
2023-12-26T18:10:18.334 INFO:teuthology.orchestra.run.smithi159.stdout:VERSION_ID="8.4"
2023-12-26T18:10:18.334 INFO:teuthology.orchestra.run.smithi159.stdout:PLATFORM_ID="platform:el8"
2023-12-26T18:10:18.335 INFO:teuthology.orchestra.run.smithi159.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.4 (Ootpa)"
2023-12-26T18:10:18.335 INFO:teuthology.orchestra.run.smithi159.stdout:ANSI_COLOR="0;31"
2023-12-26T18:10:18.335 INFO:teuthology.orchestra.run.smithi159.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8.4:GA"
2023-12-26T18:10:18.335 INFO:teuthology.orchestra.run.smithi159.stdout:HOME_URL="https://www.redhat.com/"
2023-12-26T18:10:18.335 INFO:teuthology.orchestra.run.smithi159.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/"
2023-12-26T18:10:18.335 INFO:teuthology.orchestra.run.smithi159.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/"
2023-12-26T18:10:18.335 INFO:teuthology.orchestra.run.smithi159.stdout:
2023-12-26T18:10:18.335 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8"
2023-12-26T18:10:18.335 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.4
2023-12-26T18:10:18.335 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
2023-12-26T18:10:18.335 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.4"
2023-12-26T18:10:18.336 INFO:teuthology.lock.ops:Updating smithi159.front.sepia.ceph.com on lock server
2023-12-26T18:10:18.359 DEBUG:teuthology.orchestra.run.smithi183:> uname -m
2023-12-26T18:10:18.376 INFO:teuthology.orchestra.run.smithi183.stdout:x86_64
2023-12-26T18:10:18.376 DEBUG:teuthology.orchestra.run.smithi183:> cat /etc/os-release
2023-12-26T18:10:18.432 INFO:teuthology.orchestra.run.smithi183.stdout:NAME="Red Hat Enterprise Linux"
2023-12-26T18:10:18.432 INFO:teuthology.orchestra.run.smithi183.stdout:VERSION="8.4 (Ootpa)"
2023-12-26T18:10:18.433 INFO:teuthology.orchestra.run.smithi183.stdout:ID="rhel"
2023-12-26T18:10:18.433 INFO:teuthology.orchestra.run.smithi183.stdout:ID_LIKE="fedora"
2023-12-26T18:10:18.433 INFO:teuthology.orchestra.run.smithi183.stdout:VERSION_ID="8.4"
2023-12-26T18:10:18.433 INFO:teuthology.orchestra.run.smithi183.stdout:PLATFORM_ID="platform:el8"
2023-12-26T18:10:18.433 INFO:teuthology.orchestra.run.smithi183.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.4 (Ootpa)"
2023-12-26T18:10:18.433 INFO:teuthology.orchestra.run.smithi183.stdout:ANSI_COLOR="0;31"
2023-12-26T18:10:18.433 INFO:teuthology.orchestra.run.smithi183.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8.4:GA"
2023-12-26T18:10:18.433 INFO:teuthology.orchestra.run.smithi183.stdout:HOME_URL="https://www.redhat.com/"
2023-12-26T18:10:18.433 INFO:teuthology.orchestra.run.smithi183.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/"
2023-12-26T18:10:18.433 INFO:teuthology.orchestra.run.smithi183.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/"
2023-12-26T18:10:18.433 INFO:teuthology.orchestra.run.smithi183.stdout:
2023-12-26T18:10:18.433 INFO:teuthology.orchestra.run.smithi183.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8"
2023-12-26T18:10:18.433 INFO:teuthology.orchestra.run.smithi183.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.4
2023-12-26T18:10:18.434 INFO:teuthology.orchestra.run.smithi183.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
2023-12-26T18:10:18.434 INFO:teuthology.orchestra.run.smithi183.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.4"
2023-12-26T18:10:18.434 INFO:teuthology.lock.ops:Updating smithi183.front.sepia.ceph.com on lock server
2023-12-26T18:10:18.459 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles...
2023-12-26T18:10:18.467 INFO:teuthology.run_tasks:Running task internal.check_conflict...
2023-12-26T18:10:18.485 INFO:teuthology.task.internal:Checking for old test directory...
2023-12-26T18:10:18.486 DEBUG:teuthology.orchestra.run.smithi145:> test '!' -e /home/ubuntu/cephtest
2023-12-26T18:10:18.488 DEBUG:teuthology.orchestra.run.smithi159:> test '!' -e /home/ubuntu/cephtest
2023-12-26T18:10:18.490 DEBUG:teuthology.orchestra.run.smithi183:> test '!' -e /home/ubuntu/cephtest
2023-12-26T18:10:18.505 INFO:teuthology.run_tasks:Running task internal.check_ceph_data...
2023-12-26T18:10:18.510 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph...
2023-12-26T18:10:18.511 DEBUG:teuthology.orchestra.run.smithi145:> test -z $(ls -A /var/lib/ceph)
2023-12-26T18:10:18.545 DEBUG:teuthology.orchestra.run.smithi159:> test -z $(ls -A /var/lib/ceph)
2023-12-26T18:10:18.547 DEBUG:teuthology.orchestra.run.smithi183:> test -z $(ls -A /var/lib/ceph)
2023-12-26T18:10:18.558 INFO:teuthology.orchestra.run.smithi145.stderr:ls: cannot access '/var/lib/ceph': No such file or directory
2023-12-26T18:10:18.560 INFO:teuthology.orchestra.run.smithi159.stderr:ls: cannot access '/var/lib/ceph': No such file or directory
2023-12-26T18:10:18.563 INFO:teuthology.orchestra.run.smithi183.stderr:ls: cannot access '/var/lib/ceph': No such file or directory
2023-12-26T18:10:18.564 INFO:teuthology.run_tasks:Running task internal.vm_setup...
2023-12-26T18:10:18.642 INFO:teuthology.run_tasks:Running task kernel...
2023-12-26T18:10:18.656 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'}
2023-12-26T18:10:18.656 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}, 'client.0': {'kdb': True, 'sha1': 'distro'}}, timeout 300
2023-12-26T18:10:18.657 DEBUG:teuthology.orchestra.run.smithi145:> test -f /run/.containerenv -o -f /.dockerenv
2023-12-26T18:10:18.657 DEBUG:teuthology.orchestra.run.smithi159:> test -f /run/.containerenv -o -f /.dockerenv
2023-12-26T18:10:18.657 DEBUG:teuthology.orchestra.run.smithi183:> test -f /run/.containerenv -o -f /.dockerenv
2023-12-26T18:10:18.673 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-12-26T18:10:18.673 DEBUG:teuthology.orchestra.run.smithi159:> uname -r
2023-12-26T18:10:18.675 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-12-26T18:10:18.676 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-12-26T18:10:18.677 DEBUG:teuthology.orchestra.run.smithi183:> uname -r
2023-12-26T18:10:18.677 DEBUG:teuthology.orchestra.run.smithi145:> uname -r
2023-12-26T18:10:18.729 INFO:teuthology.orchestra.run.smithi159.stdout:4.18.0-305.25.1.el8_4.x86_64
2023-12-26T18:10:18.730 INFO:teuthology.task.kernel:Running kernel on smithi159: 4.18.0-305.25.1.el8_4.x86_64
2023-12-26T18:10:18.730 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum install -y kernel
2023-12-26T18:10:18.732 INFO:teuthology.orchestra.run.smithi183.stdout:4.18.0-305.25.1.el8_4.x86_64
2023-12-26T18:10:18.732 INFO:teuthology.task.kernel:Running kernel on smithi183: 4.18.0-305.25.1.el8_4.x86_64
2023-12-26T18:10:18.732 DEBUG:teuthology.orchestra.run.smithi183:> sudo yum install -y kernel
2023-12-26T18:10:18.733 INFO:teuthology.orchestra.run.smithi145.stdout:4.18.0-305.25.1.el8_4.x86_64
2023-12-26T18:10:18.734 INFO:teuthology.task.kernel:Running kernel on smithi145: 4.18.0-305.25.1.el8_4.x86_64
2023-12-26T18:10:18.734 DEBUG:teuthology.orchestra.run.smithi145:> sudo yum install -y kernel
2023-12-26T18:10:20.179 INFO:teuthology.orchestra.run.smithi159.stdout:Updating Subscription Management repositories.
2023-12-26T18:10:20.180 INFO:teuthology.orchestra.run.smithi159.stdout:Unable to read consumer identity
2023-12-26T18:10:20.181 INFO:teuthology.orchestra.run.smithi183.stdout:Updating Subscription Management repositories.
2023-12-26T18:10:20.182 INFO:teuthology.orchestra.run.smithi183.stdout:Unable to read consumer identity
2023-12-26T18:10:20.189 INFO:teuthology.orchestra.run.smithi145.stdout:Updating Subscription Management repositories.
2023-12-26T18:10:20.190 INFO:teuthology.orchestra.run.smithi145.stdout:Unable to read consumer identity
2023-12-26T18:10:21.548 INFO:teuthology.orchestra.run.smithi159.stdout:Copr repo for python3-asyncssh owned by ceph     26 kB/s | 3.5 kB     00:00
2023-12-26T18:10:21.590 INFO:teuthology.orchestra.run.smithi145.stdout:Copr repo for python3-asyncssh owned by ceph     27 kB/s | 3.5 kB     00:00
2023-12-26T18:10:21.634 INFO:teuthology.orchestra.run.smithi183.stdout:Copr repo for python3-asyncssh owned by ceph     21 kB/s | 3.5 kB     00:00
2023-12-26T18:10:22.520 INFO:teuthology.orchestra.run.smithi159.stdout:Extra Packages for Enterprise Linux              17 MB/s |  16 MB     00:00
2023-12-26T18:10:22.754 INFO:teuthology.orchestra.run.smithi145.stdout:Extra Packages for Enterprise Linux              14 MB/s |  16 MB     00:01
2023-12-26T18:10:23.399 INFO:teuthology.orchestra.run.smithi183.stdout:Extra Packages for Enterprise Linux             9.3 MB/s |  16 MB     00:01
2023-12-26T18:10:25.880 INFO:teuthology.orchestra.run.smithi159.stdout:lab-extras                                      4.5 MB/s |  24 kB     00:00
2023-12-26T18:10:26.048 INFO:teuthology.orchestra.run.smithi145.stdout:lab-extras                                      4.1 MB/s |  24 kB     00:00
2023-12-26T18:10:26.636 INFO:teuthology.orchestra.run.smithi159.stdout:Package kernel-4.18.0-80.el8.x86_64 is already installed.
2023-12-26T18:10:26.637 INFO:teuthology.orchestra.run.smithi159.stdout:Package kernel-4.18.0-425.3.1.el8.x86_64 is already installed.
2023-12-26T18:10:26.637 INFO:teuthology.orchestra.run.smithi159.stdout:Package kernel-4.18.0-305.25.1.el8_4.x86_64 is already installed.
2023-12-26T18:10:26.745 INFO:teuthology.orchestra.run.smithi183.stdout:lab-extras                                      4.6 MB/s |  24 kB     00:00
2023-12-26T18:10:26.787 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved.
2023-12-26T18:10:26.789 INFO:teuthology.orchestra.run.smithi159.stdout:Nothing to do.
2023-12-26T18:10:26.789 INFO:teuthology.orchestra.run.smithi159.stdout:Complete!
2023-12-26T18:10:26.818 INFO:teuthology.orchestra.run.smithi145.stdout:Package kernel-4.18.0-80.el8.x86_64 is already installed.
2023-12-26T18:10:26.818 INFO:teuthology.orchestra.run.smithi145.stdout:Package kernel-4.18.0-425.3.1.el8.x86_64 is already installed.
2023-12-26T18:10:26.818 INFO:teuthology.orchestra.run.smithi145.stdout:Package kernel-4.18.0-305.25.1.el8_4.x86_64 is already installed.
2023-12-26T18:10:26.896 DEBUG:teuthology.orchestra.run.smithi159:> echo no | sudo yum reinstall kernel || true
2023-12-26T18:10:26.963 INFO:teuthology.orchestra.run.smithi145.stdout:Dependencies resolved.
2023-12-26T18:10:26.964 INFO:teuthology.orchestra.run.smithi145.stdout:Nothing to do.
2023-12-26T18:10:26.964 INFO:teuthology.orchestra.run.smithi145.stdout:Complete!
2023-12-26T18:10:27.078 DEBUG:teuthology.orchestra.run.smithi145:> echo no | sudo yum reinstall kernel || true
2023-12-26T18:10:27.244 INFO:teuthology.orchestra.run.smithi159.stdout:Updating Subscription Management repositories.
2023-12-26T18:10:27.244 INFO:teuthology.orchestra.run.smithi159.stdout:Unable to read consumer identity
2023-12-26T18:10:27.463 INFO:teuthology.orchestra.run.smithi145.stdout:Updating Subscription Management repositories.
2023-12-26T18:10:27.463 INFO:teuthology.orchestra.run.smithi145.stdout:Unable to read consumer identity
2023-12-26T18:10:27.534 INFO:teuthology.orchestra.run.smithi183.stdout:Package kernel-4.18.0-80.el8.x86_64 is already installed.
2023-12-26T18:10:27.534 INFO:teuthology.orchestra.run.smithi183.stdout:Package kernel-4.18.0-425.3.1.el8.x86_64 is already installed.
2023-12-26T18:10:27.534 INFO:teuthology.orchestra.run.smithi183.stdout:Package kernel-4.18.0-305.25.1.el8_4.x86_64 is already installed.
2023-12-26T18:10:27.545 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:00:02 ago on Tue 26 Dec 2023 06:10:25 PM UTC.
2023-12-26T18:10:27.674 INFO:teuthology.orchestra.run.smithi159.stdout:Installed package kernel-4.18.0-80.el8.x86_64 (from anaconda) not available.
2023-12-26T18:10:27.674 INFO:teuthology.orchestra.run.smithi159.stdout:Installed package kernel-4.18.0-425.3.1.el8.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2023-12-26T18:10:27.674 INFO:teuthology.orchestra.run.smithi159.stdout:Installed package kernel-4.18.0-305.25.1.el8_4.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2023-12-26T18:10:27.701 INFO:teuthology.orchestra.run.smithi183.stdout:Dependencies resolved.
2023-12-26T18:10:27.702 INFO:teuthology.orchestra.run.smithi183.stdout:Nothing to do.
2023-12-26T18:10:27.702 INFO:teuthology.orchestra.run.smithi183.stdout:Complete!
2023-12-26T18:10:27.703 INFO:teuthology.orchestra.run.smithi159.stderr:Error: No packages marked for reinstall.
2023-12-26T18:10:27.752 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum reinstall -y kernel || true
2023-12-26T18:10:27.753 INFO:teuthology.orchestra.run.smithi145.stdout:Last metadata expiration check: 0:00:01 ago on Tue 26 Dec 2023 06:10:26 PM UTC.
2023-12-26T18:10:27.810 DEBUG:teuthology.orchestra.run.smithi183:> echo no | sudo yum reinstall kernel || true
2023-12-26T18:10:27.893 INFO:teuthology.orchestra.run.smithi145.stdout:Installed package kernel-4.18.0-80.el8.x86_64 (from anaconda) not available.
2023-12-26T18:10:27.894 INFO:teuthology.orchestra.run.smithi145.stdout:Installed package kernel-4.18.0-425.3.1.el8.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2023-12-26T18:10:27.894 INFO:teuthology.orchestra.run.smithi145.stdout:Installed package kernel-4.18.0-305.25.1.el8_4.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2023-12-26T18:10:27.919 INFO:teuthology.orchestra.run.smithi145.stderr:Error: No packages marked for reinstall.
2023-12-26T18:10:27.967 DEBUG:teuthology.orchestra.run.smithi145:> sudo yum reinstall -y kernel || true
2023-12-26T18:10:28.097 INFO:teuthology.orchestra.run.smithi159.stdout:Updating Subscription Management repositories.
2023-12-26T18:10:28.100 INFO:teuthology.orchestra.run.smithi159.stdout:Unable to read consumer identity
2023-12-26T18:10:28.154 INFO:teuthology.orchestra.run.smithi183.stdout:Updating Subscription Management repositories.
2023-12-26T18:10:28.154 INFO:teuthology.orchestra.run.smithi183.stdout:Unable to read consumer identity
2023-12-26T18:10:28.312 INFO:teuthology.orchestra.run.smithi145.stdout:Updating Subscription Management repositories.
2023-12-26T18:10:28.313 INFO:teuthology.orchestra.run.smithi145.stdout:Unable to read consumer identity
2023-12-26T18:10:28.382 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:00:03 ago on Tue 26 Dec 2023 06:10:25 PM UTC.
2023-12-26T18:10:28.447 INFO:teuthology.orchestra.run.smithi183.stdout:Last metadata expiration check: 0:00:02 ago on Tue 26 Dec 2023 06:10:26 PM UTC.
2023-12-26T18:10:28.523 INFO:teuthology.orchestra.run.smithi159.stdout:Installed package kernel-4.18.0-80.el8.x86_64 (from anaconda) not available.
2023-12-26T18:10:28.523 INFO:teuthology.orchestra.run.smithi159.stdout:Installed package kernel-4.18.0-425.3.1.el8.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2023-12-26T18:10:28.523 INFO:teuthology.orchestra.run.smithi159.stdout:Installed package kernel-4.18.0-305.25.1.el8_4.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2023-12-26T18:10:28.538 INFO:teuthology.orchestra.run.smithi159.stderr:Error: No packages marked for reinstall.
2023-12-26T18:10:28.581 INFO:teuthology.orchestra.run.smithi183.stdout:Installed package kernel-4.18.0-80.el8.x86_64 (from anaconda) not available.
2023-12-26T18:10:28.581 INFO:teuthology.orchestra.run.smithi183.stdout:Installed package kernel-4.18.0-425.3.1.el8.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2023-12-26T18:10:28.581 INFO:teuthology.orchestra.run.smithi183.stdout:Installed package kernel-4.18.0-305.25.1.el8_4.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2023-12-26T18:10:28.591 DEBUG:teuthology.orchestra.run.smithi159:> rpm -q kernel | sort -rV | head -n 1
2023-12-26T18:10:28.603 INFO:teuthology.orchestra.run.smithi145.stdout:Last metadata expiration check: 0:00:02 ago on Tue 26 Dec 2023 06:10:26 PM UTC.
2023-12-26T18:10:28.613 INFO:teuthology.orchestra.run.smithi183.stderr:Error: No packages marked for reinstall.
2023-12-26T18:10:28.664 DEBUG:teuthology.orchestra.run.smithi183:> sudo yum reinstall -y kernel || true
2023-12-26T18:10:28.716 INFO:teuthology.orchestra.run.smithi159.stdout:kernel-4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:28.716 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:28.717 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-305.25.1.el8_4.x86_64 Expected: 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:28.717 INFO:teuthology.task.kernel:Skipping firmware on distro kernel
2023-12-26T18:10:28.717 DEBUG:teuthology.task.kernel:src is distro, skipping download
2023-12-26T18:10:28.717 INFO:teuthology.task.kernel:Installing distro kernel on mon.b...
2023-12-26T18:10:28.717 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi159.front.sepia.ceph.com, path=None, version=distro)
2023-12-26T18:10:28.717 DEBUG:teuthology.orchestra.run.smithi159:> rpm -q kernel | sort -rV | head -n 1
2023-12-26T18:10:28.734 INFO:teuthology.orchestra.run.smithi145.stdout:Installed package kernel-4.18.0-80.el8.x86_64 (from anaconda) not available.
2023-12-26T18:10:28.734 INFO:teuthology.orchestra.run.smithi145.stdout:Installed package kernel-4.18.0-425.3.1.el8.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2023-12-26T18:10:28.734 INFO:teuthology.orchestra.run.smithi145.stdout:Installed package kernel-4.18.0-305.25.1.el8_4.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2023-12-26T18:10:28.747 INFO:teuthology.orchestra.run.smithi145.stderr:Error: No packages marked for reinstall.
2023-12-26T18:10:28.799 DEBUG:teuthology.orchestra.run.smithi145:> rpm -q kernel | sort -rV | head -n 1
2023-12-26T18:10:28.843 INFO:teuthology.orchestra.run.smithi159.stdout:kernel-4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:28.843 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:28.843 DEBUG:teuthology.orchestra.run.smithi159:> sudo rpm -qi grub2-tools
2023-12-26T18:10:28.870 INFO:teuthology.orchestra.run.smithi159.stdout:Name        : grub2-tools
2023-12-26T18:10:28.870 INFO:teuthology.orchestra.run.smithi159.stdout:Epoch       : 1
2023-12-26T18:10:28.870 INFO:teuthology.orchestra.run.smithi159.stdout:Version     : 2.02
2023-12-26T18:10:28.870 INFO:teuthology.orchestra.run.smithi159.stdout:Release     : 99.el8_4.1
2023-12-26T18:10:28.870 INFO:teuthology.orchestra.run.smithi159.stdout:Architecture: x86_64
2023-12-26T18:10:28.870 INFO:teuthology.orchestra.run.smithi159.stdout:Install Date: Wed 21 Dec 2022 01:02:32 PM UTC
2023-12-26T18:10:28.871 INFO:teuthology.orchestra.run.smithi159.stdout:Group       : System Environment/Base
2023-12-26T18:10:28.871 INFO:teuthology.orchestra.run.smithi159.stdout:Size        : 9264631
2023-12-26T18:10:28.871 INFO:teuthology.orchestra.run.smithi159.stdout:License     : GPLv3+
2023-12-26T18:10:28.871 INFO:teuthology.orchestra.run.smithi159.stdout:Signature   : RSA/SHA256, Thu 09 Sep 2021 09:41:55 AM UTC, Key ID 199e2f91fd431d51
2023-12-26T18:10:28.871 INFO:teuthology.orchestra.run.smithi159.stdout:Source RPM  : grub2-2.02-99.el8_4.1.src.rpm
2023-12-26T18:10:28.871 INFO:teuthology.orchestra.run.smithi159.stdout:Build Date  : Tue 18 May 2021 07:46:34 AM UTC
2023-12-26T18:10:28.871 INFO:teuthology.orchestra.run.smithi159.stdout:Build Host  : x86-vm-07.build.eng.bos.redhat.com
2023-12-26T18:10:28.871 INFO:teuthology.orchestra.run.smithi159.stdout:Relocations : (not relocatable)
2023-12-26T18:10:28.872 INFO:teuthology.orchestra.run.smithi159.stdout:Packager    : Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>
2023-12-26T18:10:28.872 INFO:teuthology.orchestra.run.smithi159.stdout:Vendor      : Red Hat, Inc.
2023-12-26T18:10:28.872 INFO:teuthology.orchestra.run.smithi159.stdout:URL         : http://www.gnu.org/software/grub/
2023-12-26T18:10:28.872 INFO:teuthology.orchestra.run.smithi159.stdout:Summary     : Support tools for GRUB.
2023-12-26T18:10:28.872 INFO:teuthology.orchestra.run.smithi159.stdout:Description :
2023-12-26T18:10:28.872 INFO:teuthology.orchestra.run.smithi159.stdout:
2023-12-26T18:10:28.872 INFO:teuthology.orchestra.run.smithi159.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and
2023-12-26T18:10:28.872 INFO:teuthology.orchestra.run.smithi159.stdout:customizable bootloader with modular architecture.  It supports a rich
2023-12-26T18:10:28.872 INFO:teuthology.orchestra.run.smithi159.stdout:variety of kernel formats, file systems, computer architectures and
2023-12-26T18:10:28.872 INFO:teuthology.orchestra.run.smithi159.stdout:hardware devices.
2023-12-26T18:10:28.872 INFO:teuthology.orchestra.run.smithi159.stdout:
2023-12-26T18:10:28.872 INFO:teuthology.orchestra.run.smithi159.stdout:This subpackage provides tools for support of all platforms.
2023-12-26T18:10:28.873 INFO:teuthology.task.kernel:Updating Grub Version: grub2
2023-12-26T18:10:28.873 INFO:teuthology.task.kernel:Updating grub on smithi159 to boot 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:28.873 DEBUG:teuthology.orchestra.run.smithi159:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg
2023-12-26T18:10:28.925 INFO:teuthology.orchestra.run.smithi145.stdout:kernel-4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:28.926 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:28.926 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-305.25.1.el8_4.x86_64 Expected: 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:28.926 INFO:teuthology.task.kernel:Skipping firmware on distro kernel
2023-12-26T18:10:28.926 DEBUG:teuthology.task.kernel:src is distro, skipping download
2023-12-26T18:10:28.926 INFO:teuthology.task.kernel:Installing distro kernel on mon.a...
2023-12-26T18:10:28.926 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi145.front.sepia.ceph.com, path=None, version=distro)
2023-12-26T18:10:28.926 DEBUG:teuthology.orchestra.run.smithi145:> rpm -q kernel | sort -rV | head -n 1
2023-12-26T18:10:29.008 INFO:teuthology.orchestra.run.smithi183.stdout:Updating Subscription Management repositories.
2023-12-26T18:10:29.009 INFO:teuthology.orchestra.run.smithi183.stdout:Unable to read consumer identity
2023-12-26T18:10:29.051 INFO:teuthology.orchestra.run.smithi145.stdout:kernel-4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:29.052 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:29.052 DEBUG:teuthology.orchestra.run.smithi145:> sudo rpm -qi grub2-tools
2023-12-26T18:10:29.078 INFO:teuthology.orchestra.run.smithi145.stdout:Name        : grub2-tools
2023-12-26T18:10:29.078 INFO:teuthology.orchestra.run.smithi145.stdout:Epoch       : 1
2023-12-26T18:10:29.079 INFO:teuthology.orchestra.run.smithi145.stdout:Version     : 2.02
2023-12-26T18:10:29.079 INFO:teuthology.orchestra.run.smithi145.stdout:Release     : 99.el8_4.1
2023-12-26T18:10:29.079 INFO:teuthology.orchestra.run.smithi145.stdout:Architecture: x86_64
2023-12-26T18:10:29.079 INFO:teuthology.orchestra.run.smithi145.stdout:Install Date: Wed 21 Dec 2022 01:02:32 PM UTC
2023-12-26T18:10:29.079 INFO:teuthology.orchestra.run.smithi145.stdout:Group       : System Environment/Base
2023-12-26T18:10:29.079 INFO:teuthology.orchestra.run.smithi145.stdout:Size        : 9264631
2023-12-26T18:10:29.079 INFO:teuthology.orchestra.run.smithi145.stdout:License     : GPLv3+
2023-12-26T18:10:29.079 INFO:teuthology.orchestra.run.smithi145.stdout:Signature   : RSA/SHA256, Thu 09 Sep 2021 09:41:55 AM UTC, Key ID 199e2f91fd431d51
2023-12-26T18:10:29.079 INFO:teuthology.orchestra.run.smithi145.stdout:Source RPM  : grub2-2.02-99.el8_4.1.src.rpm
2023-12-26T18:10:29.079 INFO:teuthology.orchestra.run.smithi145.stdout:Build Date  : Tue 18 May 2021 07:46:34 AM UTC
2023-12-26T18:10:29.079 INFO:teuthology.orchestra.run.smithi145.stdout:Build Host  : x86-vm-07.build.eng.bos.redhat.com
2023-12-26T18:10:29.079 INFO:teuthology.orchestra.run.smithi145.stdout:Relocations : (not relocatable)
2023-12-26T18:10:29.079 INFO:teuthology.orchestra.run.smithi145.stdout:Packager    : Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>
2023-12-26T18:10:29.080 INFO:teuthology.orchestra.run.smithi145.stdout:Vendor      : Red Hat, Inc.
2023-12-26T18:10:29.080 INFO:teuthology.orchestra.run.smithi145.stdout:URL         : http://www.gnu.org/software/grub/
2023-12-26T18:10:29.080 INFO:teuthology.orchestra.run.smithi145.stdout:Summary     : Support tools for GRUB.
2023-12-26T18:10:29.080 INFO:teuthology.orchestra.run.smithi145.stdout:Description :
2023-12-26T18:10:29.080 INFO:teuthology.orchestra.run.smithi145.stdout:
2023-12-26T18:10:29.080 INFO:teuthology.orchestra.run.smithi145.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and
2023-12-26T18:10:29.080 INFO:teuthology.orchestra.run.smithi145.stdout:customizable bootloader with modular architecture.  It supports a rich
2023-12-26T18:10:29.080 INFO:teuthology.orchestra.run.smithi145.stdout:variety of kernel formats, file systems, computer architectures and
2023-12-26T18:10:29.080 INFO:teuthology.orchestra.run.smithi145.stdout:hardware devices.
2023-12-26T18:10:29.080 INFO:teuthology.orchestra.run.smithi145.stdout:
2023-12-26T18:10:29.080 INFO:teuthology.orchestra.run.smithi145.stdout:This subpackage provides tools for support of all platforms.
2023-12-26T18:10:29.081 INFO:teuthology.task.kernel:Updating Grub Version: grub2
2023-12-26T18:10:29.081 INFO:teuthology.task.kernel:Updating grub on smithi145 to boot 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:29.081 DEBUG:teuthology.orchestra.run.smithi145:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg
2023-12-26T18:10:29.299 INFO:teuthology.orchestra.run.smithi183.stdout:Last metadata expiration check: 0:00:03 ago on Tue 26 Dec 2023 06:10:26 PM UTC.
2023-12-26T18:10:29.440 INFO:teuthology.orchestra.run.smithi183.stdout:Installed package kernel-4.18.0-80.el8.x86_64 (from anaconda) not available.
2023-12-26T18:10:29.440 INFO:teuthology.orchestra.run.smithi183.stdout:Installed package kernel-4.18.0-425.3.1.el8.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2023-12-26T18:10:29.441 INFO:teuthology.orchestra.run.smithi183.stdout:Installed package kernel-4.18.0-305.25.1.el8_4.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2023-12-26T18:10:29.453 INFO:teuthology.orchestra.run.smithi183.stderr:Error: No packages marked for reinstall.
2023-12-26T18:10:29.504 DEBUG:teuthology.orchestra.run.smithi183:> rpm -q kernel | sort -rV | head -n 1
2023-12-26T18:10:29.628 INFO:teuthology.orchestra.run.smithi183.stdout:kernel-4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:29.629 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:29.629 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-305.25.1.el8_4.x86_64 Expected: 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:29.629 INFO:teuthology.task.kernel:Skipping firmware on distro kernel
2023-12-26T18:10:29.629 DEBUG:teuthology.task.kernel:src is distro, skipping download
2023-12-26T18:10:29.629 INFO:teuthology.task.kernel:Installing distro kernel on client.0...
2023-12-26T18:10:29.629 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi183.front.sepia.ceph.com, path=None, version=distro)
2023-12-26T18:10:29.629 DEBUG:teuthology.orchestra.run.smithi183:> rpm -q kernel | sort -rV | head -n 1
2023-12-26T18:10:29.756 INFO:teuthology.orchestra.run.smithi183.stdout:kernel-4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:29.756 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:29.756 DEBUG:teuthology.orchestra.run.smithi183:> sudo rpm -qi grub2-tools
2023-12-26T18:10:29.783 INFO:teuthology.orchestra.run.smithi183.stdout:Name        : grub2-tools
2023-12-26T18:10:29.783 INFO:teuthology.orchestra.run.smithi183.stdout:Epoch       : 1
2023-12-26T18:10:29.783 INFO:teuthology.orchestra.run.smithi183.stdout:Version     : 2.02
2023-12-26T18:10:29.783 INFO:teuthology.orchestra.run.smithi183.stdout:Release     : 99.el8_4.1
2023-12-26T18:10:29.783 INFO:teuthology.orchestra.run.smithi183.stdout:Architecture: x86_64
2023-12-26T18:10:29.784 INFO:teuthology.orchestra.run.smithi183.stdout:Install Date: Wed 21 Dec 2022 01:02:32 PM UTC
2023-12-26T18:10:29.784 INFO:teuthology.orchestra.run.smithi183.stdout:Group       : System Environment/Base
2023-12-26T18:10:29.784 INFO:teuthology.orchestra.run.smithi183.stdout:Size        : 9264631
2023-12-26T18:10:29.784 INFO:teuthology.orchestra.run.smithi183.stdout:License     : GPLv3+
2023-12-26T18:10:29.784 INFO:teuthology.orchestra.run.smithi183.stdout:Signature   : RSA/SHA256, Thu 09 Sep 2021 09:41:55 AM UTC, Key ID 199e2f91fd431d51
2023-12-26T18:10:29.784 INFO:teuthology.orchestra.run.smithi183.stdout:Source RPM  : grub2-2.02-99.el8_4.1.src.rpm
2023-12-26T18:10:29.784 INFO:teuthology.orchestra.run.smithi183.stdout:Build Date  : Tue 18 May 2021 07:46:34 AM UTC
2023-12-26T18:10:29.784 INFO:teuthology.orchestra.run.smithi183.stdout:Build Host  : x86-vm-07.build.eng.bos.redhat.com
2023-12-26T18:10:29.784 INFO:teuthology.orchestra.run.smithi183.stdout:Relocations : (not relocatable)
2023-12-26T18:10:29.784 INFO:teuthology.orchestra.run.smithi183.stdout:Packager    : Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>
2023-12-26T18:10:29.784 INFO:teuthology.orchestra.run.smithi183.stdout:Vendor      : Red Hat, Inc.
2023-12-26T18:10:29.784 INFO:teuthology.orchestra.run.smithi183.stdout:URL         : http://www.gnu.org/software/grub/
2023-12-26T18:10:29.784 INFO:teuthology.orchestra.run.smithi183.stdout:Summary     : Support tools for GRUB.
2023-12-26T18:10:29.785 INFO:teuthology.orchestra.run.smithi183.stdout:Description :
2023-12-26T18:10:29.785 INFO:teuthology.orchestra.run.smithi183.stdout:
2023-12-26T18:10:29.785 INFO:teuthology.orchestra.run.smithi183.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and
2023-12-26T18:10:29.785 INFO:teuthology.orchestra.run.smithi183.stdout:customizable bootloader with modular architecture.  It supports a rich
2023-12-26T18:10:29.785 INFO:teuthology.orchestra.run.smithi183.stdout:variety of kernel formats, file systems, computer architectures and
2023-12-26T18:10:29.785 INFO:teuthology.orchestra.run.smithi183.stdout:hardware devices.
2023-12-26T18:10:29.785 INFO:teuthology.orchestra.run.smithi183.stdout:
2023-12-26T18:10:29.785 INFO:teuthology.orchestra.run.smithi183.stdout:This subpackage provides tools for support of all platforms.
2023-12-26T18:10:29.786 INFO:teuthology.task.kernel:Updating Grub Version: grub2
2023-12-26T18:10:29.786 INFO:teuthology.task.kernel:Updating grub on smithi183 to boot 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:29.786 DEBUG:teuthology.orchestra.run.smithi183:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg
2023-12-26T18:10:30.265 INFO:teuthology.orchestra.run.smithi159.stderr:Generating grub configuration file ...
2023-12-26T18:10:30.335 INFO:teuthology.orchestra.run.smithi145.stderr:Generating grub configuration file ...
2023-12-26T18:10:31.114 INFO:teuthology.orchestra.run.smithi183.stderr:Generating grub configuration file ...
2023-12-26T18:10:32.074 INFO:teuthology.orchestra.run.smithi159.stderr:done
2023-12-26T18:10:32.076 DEBUG:teuthology.orchestra.run.smithi159:> mktemp
2023-12-26T18:10:32.106 INFO:teuthology.orchestra.run.smithi159.stdout:/tmp/tmp.bqoCm83Zj7
2023-12-26T18:10:32.107 DEBUG:teuthology.orchestra.run.smithi159:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.bqoCm83Zj7
2023-12-26T18:10:32.189 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod 0666 /tmp/tmp.bqoCm83Zj7
2023-12-26T18:10:32.287 INFO:teuthology.orchestra.run.smithi145.stderr:done
2023-12-26T18:10:32.289 DEBUG:teuthology.orchestra.run.smithi145:> mktemp
2023-12-26T18:10:32.303 INFO:teuthology.orchestra.run.smithi145.stdout:/tmp/tmp.5MRxez7BJR
2023-12-26T18:10:32.303 DEBUG:teuthology.orchestra.run.smithi145:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.5MRxez7BJR
2023-12-26T18:10:32.366 DEBUG:teuthology.orchestra.remote:smithi159:/tmp/tmp.bqoCm83Zj7 is 6KB
2023-12-26T18:10:32.372 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod 0666 /tmp/tmp.5MRxez7BJR
2023-12-26T18:10:32.415 DEBUG:teuthology.orchestra.run.smithi159:> rm -fr /tmp/tmp.bqoCm83Zj7
2023-12-26T18:10:32.428 DEBUG:teuthology.orchestra.run.smithi159:> sudo /bin/ls /boot/loader/entries || true
2023-12-26T18:10:32.496 INFO:teuthology.orchestra.run.smithi159.stdout:298e228f3b5e401e8ac0f5ae159a2e06-0-rescue.conf
2023-12-26T18:10:32.496 INFO:teuthology.orchestra.run.smithi159.stdout:298e228f3b5e401e8ac0f5ae159a2e06-4.18.0-305.25.1.el8_4.x86_64.conf
2023-12-26T18:10:32.496 INFO:teuthology.orchestra.run.smithi159.stdout:298e228f3b5e401e8ac0f5ae159a2e06-4.18.0-425.3.1.el8.x86_64.conf
2023-12-26T18:10:32.496 INFO:teuthology.orchestra.run.smithi159.stdout:298e228f3b5e401e8ac0f5ae159a2e06-4.18.0-80.el8.x86_64.conf
2023-12-26T18:10:32.497 DEBUG:teuthology.orchestra.run.smithi159:> sudo grub2-set-default 298e228f3b5e401e8ac0f5ae159a2e06-4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:32.525 DEBUG:teuthology.orchestra.remote:smithi145:/tmp/tmp.5MRxez7BJR is 6KB
2023-12-26T18:10:32.573 DEBUG:teuthology.orchestra.run.smithi145:> rm -fr /tmp/tmp.5MRxez7BJR
2023-12-26T18:10:32.587 DEBUG:teuthology.orchestra.run.smithi145:> sudo /bin/ls /boot/loader/entries || true
2023-12-26T18:10:32.649 DEBUG:teuthology.orchestra.run.smithi159:> sudo shutdown -r now
2023-12-26T18:10:32.662 INFO:teuthology.orchestra.run.smithi145.stdout:298e228f3b5e401e8ac0f5ae159a2e06-0-rescue.conf
2023-12-26T18:10:32.662 INFO:teuthology.orchestra.run.smithi145.stdout:298e228f3b5e401e8ac0f5ae159a2e06-4.18.0-305.25.1.el8_4.x86_64.conf
2023-12-26T18:10:32.663 INFO:teuthology.orchestra.run.smithi145.stdout:298e228f3b5e401e8ac0f5ae159a2e06-4.18.0-425.3.1.el8.x86_64.conf
2023-12-26T18:10:32.663 INFO:teuthology.orchestra.run.smithi145.stdout:298e228f3b5e401e8ac0f5ae159a2e06-4.18.0-80.el8.x86_64.conf
2023-12-26T18:10:32.664 DEBUG:teuthology.orchestra.run.smithi145:> sudo grub2-set-default 298e228f3b5e401e8ac0f5ae159a2e06-4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:32.839 DEBUG:teuthology.orchestra.run.smithi145:> sudo shutdown -r now
2023-12-26T18:10:33.040 INFO:teuthology.orchestra.run.smithi183.stderr:done
2023-12-26T18:10:33.042 DEBUG:teuthology.orchestra.run.smithi183:> mktemp
2023-12-26T18:10:33.055 INFO:teuthology.orchestra.run.smithi183.stdout:/tmp/tmp.MVUZrHnxHX
2023-12-26T18:10:33.056 DEBUG:teuthology.orchestra.run.smithi183:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.MVUZrHnxHX
2023-12-26T18:10:33.133 DEBUG:teuthology.orchestra.run.smithi183:> sudo chmod 0666 /tmp/tmp.MVUZrHnxHX
2023-12-26T18:10:33.299 DEBUG:teuthology.orchestra.remote:smithi183:/tmp/tmp.MVUZrHnxHX is 6KB
2023-12-26T18:10:33.348 DEBUG:teuthology.orchestra.run.smithi183:> rm -fr /tmp/tmp.MVUZrHnxHX
2023-12-26T18:10:33.362 DEBUG:teuthology.orchestra.run.smithi183:> sudo /bin/ls /boot/loader/entries || true
2023-12-26T18:10:33.441 INFO:teuthology.orchestra.run.smithi183.stdout:298e228f3b5e401e8ac0f5ae159a2e06-0-rescue.conf
2023-12-26T18:10:33.442 INFO:teuthology.orchestra.run.smithi183.stdout:298e228f3b5e401e8ac0f5ae159a2e06-4.18.0-305.25.1.el8_4.x86_64.conf
2023-12-26T18:10:33.442 INFO:teuthology.orchestra.run.smithi183.stdout:298e228f3b5e401e8ac0f5ae159a2e06-4.18.0-425.3.1.el8.x86_64.conf
2023-12-26T18:10:33.442 INFO:teuthology.orchestra.run.smithi183.stdout:298e228f3b5e401e8ac0f5ae159a2e06-4.18.0-80.el8.x86_64.conf
2023-12-26T18:10:33.442 DEBUG:teuthology.orchestra.run.smithi183:> sudo grub2-set-default 298e228f3b5e401e8ac0f5ae159a2e06-4.18.0-425.3.1.el8.x86_64
2023-12-26T18:10:33.613 DEBUG:teuthology.orchestra.run.smithi183:> sudo shutdown -r now
2023-12-26T18:11:02.681 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version...
2023-12-26T18:11:02.682 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com'
2023-12-26T18:11:02.683 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-12-26T18:11:02.842 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version...
2023-12-26T18:11:02.842 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com'
2023-12-26T18:11:02.843 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-12-26T18:11:03.616 INFO:teuthology.task.kernel:Checking client client.0 for new kernel version...
2023-12-26T18:11:03.616 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi183.front.sepia.ceph.com'
2023-12-26T18:11:03.617 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi183.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-12-26T18:11:37.366 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.145
2023-12-26T18:11:37.367 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.159
2023-12-26T18:11:46.373 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com'
2023-12-26T18:11:46.374 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-12-26T18:11:46.376 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com'
2023-12-26T18:11:46.376 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-12-26T18:12:03.634 DEBUG:teuthology.orchestra.remote:timed out
2023-12-26T18:12:04.757 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.145
2023-12-26T18:12:04.758 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.159
2023-12-26T18:12:12.641 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi183.front.sepia.ceph.com'
2023-12-26T18:12:12.642 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi183.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-12-26T18:12:13.009 DEBUG:teuthology.orchestra.run.smithi183:> true
2023-12-26T18:12:13.435 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi183.front.sepia.ceph.com'
2023-12-26T18:12:13.435 INFO:teuthology.task.kernel:Checking kernel version of client.0, want "4.18.0-425.3.1.el8.x86_64"...
2023-12-26T18:12:13.436 DEBUG:teuthology.orchestra.run.smithi183:> uname -r
2023-12-26T18:12:13.449 INFO:teuthology.orchestra.run.smithi183.stdout:4.18.0-425.3.1.el8.x86_64
2023-12-26T18:12:13.450 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-425.3.1.el8.x86_64 vs 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:12:13.450 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install
2023-12-26T18:12:13.450 DEBUG:teuthology.task.kernel:Distro of this test job: rhel
2023-12-26T18:12:14.451 INFO:teuthology.task.kernel:Enabling kdb on client.0...
2023-12-26T18:12:14.451 DEBUG:teuthology.orchestra.run.smithi183:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2023-12-26T18:12:14.634 INFO:teuthology.orchestra.run.smithi183.stdout:ttyS1
2023-12-26T18:12:14.653 DEBUG:teuthology.parallel:result is None
2023-12-26T18:12:16.758 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com'
2023-12-26T18:12:16.759 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-12-26T18:12:16.760 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com'
2023-12-26T18:12:16.760 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2023-12-26T18:12:17.077 DEBUG:teuthology.orchestra.run.smithi145:> true
2023-12-26T18:12:17.314 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi145.front.sepia.ceph.com'
2023-12-26T18:12:17.314 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-425.3.1.el8.x86_64"...
2023-12-26T18:12:17.314 DEBUG:teuthology.orchestra.run.smithi145:> uname -r
2023-12-26T18:12:17.328 INFO:teuthology.orchestra.run.smithi145.stdout:4.18.0-425.3.1.el8.x86_64
2023-12-26T18:12:17.328 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-425.3.1.el8.x86_64 vs 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:12:17.328 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install
2023-12-26T18:12:17.328 DEBUG:teuthology.task.kernel:Distro of this test job: rhel
2023-12-26T18:12:18.329 INFO:teuthology.task.kernel:Enabling kdb on mon.a...
2023-12-26T18:12:18.330 DEBUG:teuthology.orchestra.run.smithi145:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2023-12-26T18:12:18.409 INFO:teuthology.orchestra.run.smithi145.stdout:ttyS1
2023-12-26T18:12:18.424 DEBUG:teuthology.parallel:result is None
2023-12-26T18:12:18.620 DEBUG:teuthology.orchestra.run.smithi159:> true
2023-12-26T18:12:18.847 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi159.front.sepia.ceph.com'
2023-12-26T18:12:18.847 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-425.3.1.el8.x86_64"...
2023-12-26T18:12:18.847 DEBUG:teuthology.orchestra.run.smithi159:> uname -r
2023-12-26T18:12:18.904 INFO:teuthology.orchestra.run.smithi159.stdout:4.18.0-425.3.1.el8.x86_64
2023-12-26T18:12:18.904 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-425.3.1.el8.x86_64 vs 4.18.0-425.3.1.el8.x86_64
2023-12-26T18:12:18.904 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install
2023-12-26T18:12:18.904 DEBUG:teuthology.task.kernel:Distro of this test job: rhel
2023-12-26T18:12:19.905 INFO:teuthology.task.kernel:Enabling kdb on mon.b...
2023-12-26T18:12:19.906 DEBUG:teuthology.orchestra.run.smithi159:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2023-12-26T18:12:19.992 INFO:teuthology.orchestra.run.smithi159.stdout:ttyS1
2023-12-26T18:12:20.006 DEBUG:teuthology.parallel:result is None
2023-12-26T18:12:20.006 INFO:teuthology.run_tasks:Running task internal.base...
2023-12-26T18:12:20.017 INFO:teuthology.task.internal:Creating test directory...
2023-12-26T18:12:20.018 DEBUG:teuthology.orchestra.run.smithi145:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2023-12-26T18:12:20.020 DEBUG:teuthology.orchestra.run.smithi159:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2023-12-26T18:12:20.023 DEBUG:teuthology.orchestra.run.smithi183:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2023-12-26T18:12:20.302 INFO:teuthology.run_tasks:Running task internal.archive_upload...
2023-12-26T18:12:20.311 INFO:teuthology.run_tasks:Running task internal.archive...
2023-12-26T18:12:20.447 INFO:teuthology.task.internal:Creating archive directory...
2023-12-26T18:12:20.448 DEBUG:teuthology.orchestra.run.smithi145:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2023-12-26T18:12:20.451 DEBUG:teuthology.orchestra.run.smithi159:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2023-12-26T18:12:20.454 DEBUG:teuthology.orchestra.run.smithi183:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2023-12-26T18:12:20.817 INFO:teuthology.run_tasks:Running task internal.coredump...
2023-12-26T18:12:20.828 INFO:teuthology.task.internal:Enabling coredump saving...
2023-12-26T18:12:20.828 DEBUG:teuthology.orchestra.run.smithi145:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2023-12-26T18:12:20.831 DEBUG:teuthology.orchestra.run.smithi159:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2023-12-26T18:12:20.834 DEBUG:teuthology.orchestra.run.smithi183:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2023-12-26T18:12:20.857 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2023-12-26T18:12:20.868 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2023-12-26T18:12:20.870 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2023-12-26T18:12:20.883 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2023-12-26T18:12:21.340 INFO:teuthology.orchestra.run.smithi183.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2023-12-26T18:12:21.604 INFO:teuthology.orchestra.run.smithi183.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2023-12-26T18:12:21.609 INFO:teuthology.run_tasks:Running task internal.sudo...
2023-12-26T18:12:21.617 INFO:teuthology.task.internal:Configuring sudo...
2023-12-26T18:12:21.617 DEBUG:teuthology.orchestra.run.smithi145:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2023-12-26T18:12:21.674 DEBUG:teuthology.orchestra.run.smithi159:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2023-12-26T18:12:21.676 DEBUG:teuthology.orchestra.run.smithi183:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2023-12-26T18:12:22.091 INFO:teuthology.run_tasks:Running task internal.syslog...
2023-12-26T18:12:22.101 INFO:teuthology.task.internal.syslog:Starting syslog monitoring...
2023-12-26T18:12:22.102 DEBUG:teuthology.orchestra.run.smithi145:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2023-12-26T18:12:22.151 DEBUG:teuthology.orchestra.run.smithi159:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2023-12-26T18:12:22.153 DEBUG:teuthology.orchestra.run.smithi183:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2023-12-26T18:12:22.354 DEBUG:teuthology.orchestra.run.smithi145:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2023-12-26T18:12:22.642 DEBUG:teuthology.orchestra.run.smithi145:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log
2023-12-26T18:12:23.060 DEBUG:teuthology.orchestra.run.smithi145:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2023-12-26T18:12:23.376 DEBUG:teuthology.orchestra.run.smithi145:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log
2023-12-26T18:12:23.489 DEBUG:teuthology.orchestra.run.smithi145:> set -ex
2023-12-26T18:12:23.489 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2023-12-26T18:12:23.516 DEBUG:teuthology.orchestra.run.smithi159:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2023-12-26T18:12:23.553 DEBUG:teuthology.orchestra.run.smithi159:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log
2023-12-26T18:12:23.611 DEBUG:teuthology.orchestra.run.smithi159:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2023-12-26T18:12:23.694 DEBUG:teuthology.orchestra.run.smithi159:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log
2023-12-26T18:12:23.728 DEBUG:teuthology.orchestra.run.smithi159:> set -ex
2023-12-26T18:12:23.728 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2023-12-26T18:12:23.795 DEBUG:teuthology.orchestra.run.smithi183:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2023-12-26T18:12:23.841 DEBUG:teuthology.orchestra.run.smithi183:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log
2023-12-26T18:12:23.885 DEBUG:teuthology.orchestra.run.smithi183:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2023-12-26T18:12:23.967 DEBUG:teuthology.orchestra.run.smithi183:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log
2023-12-26T18:12:24.010 DEBUG:teuthology.orchestra.run.smithi183:> set -ex
2023-12-26T18:12:24.010 DEBUG:teuthology.orchestra.run.smithi183:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2023-12-26T18:12:24.076 DEBUG:teuthology.orchestra.run.smithi145:> sudo service rsyslog restart
2023-12-26T18:12:24.079 DEBUG:teuthology.orchestra.run.smithi159:> sudo service rsyslog restart
2023-12-26T18:12:24.081 DEBUG:teuthology.orchestra.run.smithi183:> sudo service rsyslog restart
2023-12-26T18:12:24.124 INFO:teuthology.orchestra.run.smithi159.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2023-12-26T18:12:24.166 INFO:teuthology.orchestra.run.smithi183.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2023-12-26T18:12:24.376 INFO:teuthology.orchestra.run.smithi145.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2023-12-26T18:12:24.475 INFO:teuthology.run_tasks:Running task internal.timer...
2023-12-26T18:12:24.487 INFO:teuthology.task.internal:Starting timer...
2023-12-26T18:12:24.487 INFO:teuthology.run_tasks:Running task pcp...
2023-12-26T18:12:24.528 INFO:teuthology.run_tasks:Running task selinux...
2023-12-26T18:12:24.576 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}
2023-12-26T18:12:24.576 DEBUG:teuthology.orchestra.run.smithi145:> sudo service auditd rotate
2023-12-26T18:12:24.624 INFO:teuthology.orchestra.run.smithi145.stdout:Rotating logs: [  OK  ]
2023-12-26T18:12:24.625 DEBUG:teuthology.orchestra.run.smithi159:> sudo service auditd rotate
2023-12-26T18:12:24.690 INFO:teuthology.orchestra.run.smithi159.stdout:Rotating logs: [  OK  ]
2023-12-26T18:12:24.691 DEBUG:teuthology.orchestra.run.smithi183:> sudo service auditd rotate
2023-12-26T18:12:24.755 INFO:teuthology.orchestra.run.smithi183.stdout:Rotating logs: [  OK  ]
2023-12-26T18:12:24.757 DEBUG:teuthology.task.selinux:Getting current SELinux state
2023-12-26T18:12:24.757 DEBUG:teuthology.orchestra.run.smithi145:> /usr/sbin/getenforce
2023-12-26T18:12:24.775 INFO:teuthology.orchestra.run.smithi145.stdout:Permissive
2023-12-26T18:12:24.776 DEBUG:teuthology.orchestra.run.smithi159:> /usr/sbin/getenforce
2023-12-26T18:12:24.800 INFO:teuthology.orchestra.run.smithi159.stdout:Permissive
2023-12-26T18:12:24.801 DEBUG:teuthology.orchestra.run.smithi183:> /usr/sbin/getenforce
2023-12-26T18:12:24.819 INFO:teuthology.orchestra.run.smithi183.stdout:Permissive
2023-12-26T18:12:24.820 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi145.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi159.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi183.front.sepia.ceph.com': 'permissive'}
2023-12-26T18:12:24.820 DEBUG:teuthology.orchestra.run.smithi145:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2023-12-26T18:12:24.845 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-12-26T18:12:24.845 DEBUG:teuthology.orchestra.run.smithi159:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2023-12-26T18:12:24.870 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-12-26T18:12:24.872 DEBUG:teuthology.orchestra.run.smithi183:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2023-12-26T18:12:24.896 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-12-26T18:12:24.897 INFO:teuthology.task.selinux:Putting SELinux into permissive mode
2023-12-26T18:12:24.897 DEBUG:teuthology.orchestra.run.smithi145:> sudo /usr/sbin/setenforce permissive
2023-12-26T18:12:24.922 DEBUG:teuthology.orchestra.run.smithi159:> sudo /usr/sbin/setenforce permissive
2023-12-26T18:12:24.946 DEBUG:teuthology.orchestra.run.smithi183:> sudo /usr/sbin/setenforce permissive
2023-12-26T18:12:24.975 INFO:teuthology.run_tasks:Running task ansible.cephlab...
2023-12-26T18:12:24.988 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main
2023-12-26T18:12:25.009 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}]
2023-12-26T18:12:25.010 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi145.front.sepia.ceph.com,smithi159.front.sepia.ceph.com,smithi183.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml
2023-12-26T18:17:50.606 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi/7501638/ansible_failures.yaml
2023-12-26T18:17:50.608 ERROR:teuthology.run_tasks:Saw exception from tasks.
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_9680d2aba1e76f64ade295c81599c97907457848/teuthology/run_tasks.py", line 109, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9680d2aba1e76f64ade295c81599c97907457848/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9680d2aba1e76f64ade295c81599c97907457848/teuthology/task/ansible.py", line 503, in begin
    super(CephLab, self).begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9680d2aba1e76f64ade295c81599c97907457848/teuthology/task/ansible.py", line 341, in begin
    self.execute_playbook()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9680d2aba1e76f64ade295c81599c97907457848/teuthology/task/ansible.py", line 369, in execute_playbook
    self._handle_failure(command, status)
  File "/home/teuthworker/src/git.ceph.com_teuthology_9680d2aba1e76f64ade295c81599c97907457848/teuthology/task/ansible.py", line 397, in _handle_failure
    raise AnsibleFailedError(failures)
teuthology.exceptions.AnsibleFailedError: smithi159.front.sepia.ceph.com:  _ansible_no_log: false  changed: false  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - krb5-workstation      releasever: null      security: false      skip_broken: false      state: present      update_cache: false      update_only: false      validate_certs: true  msg: 'Failed to download metadata for repo ''codeready-builder-for-rhel-8-x86_64-rpms'':    Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were    tried'  rc: 1  results: []smithi183.front.sepia.ceph.com:  _ansible_no_log: false  changed: false  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - krb5-workstation      releasever: null      security: false      skip_broken: false      state: present      update_cache: false      update_only: false      validate_certs: true  msg: 'Failed to download metadata for repo ''codeready-builder-for-rhel-8-x86_64-rpms'':    Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were    tried'  rc: 1  results: []smithi145.front.sepia.ceph.com:  _ansible_no_log: false  changed: false  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - krb5-workstation      releasever: null      security: false      skip_broken: false      state: present      update_cache: false      update_only: false      validate_certs: true  msg: 'Failed to download metadata for repo ''codeready-builder-for-rhel-8-x86_64-rpms'':    Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were    tried'  rc: 1  results: []
2023-12-26T18:17:50.692 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=4c7831b67f894189881398d5f3dae705
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_9680d2aba1e76f64ade295c81599c97907457848/teuthology/run_tasks.py", line 109, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9680d2aba1e76f64ade295c81599c97907457848/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9680d2aba1e76f64ade295c81599c97907457848/teuthology/task/ansible.py", line 503, in begin
    super(CephLab, self).begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9680d2aba1e76f64ade295c81599c97907457848/teuthology/task/ansible.py", line 341, in begin
    self.execute_playbook()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9680d2aba1e76f64ade295c81599c97907457848/teuthology/task/ansible.py", line 369, in execute_playbook
    self._handle_failure(command, status)
  File "/home/teuthworker/src/git.ceph.com_teuthology_9680d2aba1e76f64ade295c81599c97907457848/teuthology/task/ansible.py", line 397, in _handle_failure
    raise AnsibleFailedError(failures)
teuthology.exceptions.AnsibleFailedError: smithi159.front.sepia.ceph.com:  _ansible_no_log: false  changed: false  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - krb5-workstation      releasever: null      security: false      skip_broken: false      state: present      update_cache: false      update_only: false      validate_certs: true  msg: 'Failed to download metadata for repo ''codeready-builder-for-rhel-8-x86_64-rpms'':    Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were    tried'  rc: 1  results: []smithi183.front.sepia.ceph.com:  _ansible_no_log: false  changed: false  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - krb5-workstation      releasever: null      security: false      skip_broken: false      state: present      update_cache: false      update_only: false      validate_certs: true  msg: 'Failed to download metadata for repo ''codeready-builder-for-rhel-8-x86_64-rpms'':    Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were    tried'  rc: 1  results: []smithi145.front.sepia.ceph.com:  _ansible_no_log: false  changed: false  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - krb5-workstation      releasever: null      security: false      skip_broken: false      state: present      update_cache: false      update_only: false      validate_certs: true  msg: 'Failed to download metadata for repo ''codeready-builder-for-rhel-8-x86_64-rpms'':    Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were    tried'  rc: 1  results: []
2023-12-26T18:17:50.694 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab
2023-12-26T18:17:50.703 INFO:teuthology.task.ansible:Skipping ansible cleanup...
2023-12-26T18:17:50.703 DEBUG:teuthology.run_tasks:Unwinding manager selinux
2023-12-26T18:17:50.744 DEBUG:teuthology.orchestra.run.smithi145:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log
2023-12-26T18:17:50.824 DEBUG:teuthology.orchestra.run.smithi159:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log
2023-12-26T18:17:50.905 DEBUG:teuthology.orchestra.run.smithi183:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log
2023-12-26T18:17:50.988 DEBUG:teuthology.orchestra.run.smithi145:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2023-12-26T18:17:51.016 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-12-26T18:17:51.017 DEBUG:teuthology.orchestra.run.smithi159:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2023-12-26T18:17:51.047 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-12-26T18:17:51.048 DEBUG:teuthology.orchestra.run.smithi183:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2023-12-26T18:17:51.077 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-12-26T18:17:51.078 DEBUG:teuthology.run_tasks:Unwinding manager pcp
2023-12-26T18:17:51.086 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer
2023-12-26T18:17:51.110 INFO:teuthology.task.internal:Duration was 326.623128 seconds
2023-12-26T18:17:51.111 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog
2023-12-26T18:17:51.161 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring...
2023-12-26T18:17:51.161 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart
2023-12-26T18:17:51.163 DEBUG:teuthology.orchestra.run.smithi159:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart
2023-12-26T18:17:51.166 DEBUG:teuthology.orchestra.run.smithi183:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart
2023-12-26T18:17:51.205 INFO:teuthology.orchestra.run.smithi145.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2023-12-26T18:17:51.210 INFO:teuthology.orchestra.run.smithi159.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2023-12-26T18:17:51.210 INFO:teuthology.orchestra.run.smithi183.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2023-12-26T18:17:51.662 INFO:teuthology.task.internal.syslog:Checking logs for errors...
2023-12-26T18:17:51.663 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi145.front.sepia.ceph.com
2023-12-26T18:17:51.663 DEBUG:teuthology.orchestra.run.smithi145:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1
2023-12-26T18:17:51.721 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi159.front.sepia.ceph.com
2023-12-26T18:17:51.722 DEBUG:teuthology.orchestra.run.smithi159:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1
2023-12-26T18:17:51.742 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi183.front.sepia.ceph.com
2023-12-26T18:17:51.742 DEBUG:teuthology.orchestra.run.smithi183:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1
2023-12-26T18:17:51.761 INFO:teuthology.task.internal.syslog:Compressing syslogs...
2023-12-26T18:17:51.762 DEBUG:teuthology.orchestra.run.smithi145:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
2023-12-26T18:17:51.764 DEBUG:teuthology.orchestra.run.smithi159:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
2023-12-26T18:17:51.785 DEBUG:teuthology.orchestra.run.smithi183:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
2023-12-26T18:17:51.836 INFO:teuthology.task.internal.syslog:Gathering journactl -b0...
2023-12-26T18:17:51.836 DEBUG:teuthology.orchestra.run.smithi145:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz
2023-12-26T18:17:51.838 DEBUG:teuthology.orchestra.run.smithi159:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz
2023-12-26T18:17:51.862 DEBUG:teuthology.orchestra.run.smithi183:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz
2023-12-26T18:17:51.935 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo
2023-12-26T18:17:51.950 INFO:teuthology.task.internal:Restoring /etc/sudoers...
2023-12-26T18:17:51.950 DEBUG:teuthology.orchestra.run.smithi145:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers
2023-12-26T18:17:51.975 DEBUG:teuthology.orchestra.run.smithi159:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers
2023-12-26T18:17:51.999 DEBUG:teuthology.orchestra.run.smithi183:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers
2023-12-26T18:17:52.024 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
2023-12-26T18:17:52.033 DEBUG:teuthology.orchestra.run.smithi145:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
2023-12-26T18:17:52.036 DEBUG:teuthology.orchestra.run.smithi159:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
2023-12-26T18:17:52.042 DEBUG:teuthology.orchestra.run.smithi183:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
2023-12-26T18:17:52.057 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern = core
2023-12-26T18:17:52.063 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern = core
2023-12-26T18:17:52.087 INFO:teuthology.orchestra.run.smithi183.stdout:kernel.core_pattern = core
2023-12-26T18:17:52.100 DEBUG:teuthology.orchestra.run.smithi145:> test -e /home/ubuntu/cephtest/archive/coredump
2023-12-26T18:17:52.124 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-12-26T18:17:52.124 DEBUG:teuthology.orchestra.run.smithi159:> test -e /home/ubuntu/cephtest/archive/coredump
2023-12-26T18:17:52.138 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-12-26T18:17:52.139 DEBUG:teuthology.orchestra.run.smithi183:> test -e /home/ubuntu/cephtest/archive/coredump
2023-12-26T18:17:52.154 DEBUG:teuthology.orchestra.run:got remote process result: 1
2023-12-26T18:17:52.155 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive
2023-12-26T18:17:52.164 INFO:teuthology.task.internal:Transferring archived files...
2023-12-26T18:17:52.166 DEBUG:teuthology.misc:Transferring archived files from smithi145:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi/7501638/remote/smithi145
2023-12-26T18:17:52.167 DEBUG:teuthology.orchestra.run.smithi145:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- .
2023-12-26T18:17:52.215 DEBUG:teuthology.misc:Transferring archived files from smithi159:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi/7501638/remote/smithi159
2023-12-26T18:17:52.216 DEBUG:teuthology.orchestra.run.smithi159:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- .
2023-12-26T18:17:52.259 DEBUG:teuthology.misc:Transferring archived files from smithi183:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi/7501638/remote/smithi183
2023-12-26T18:17:52.260 DEBUG:teuthology.orchestra.run.smithi183:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- .
2023-12-26T18:17:52.308 INFO:teuthology.task.internal:Removing archive directory...
2023-12-26T18:17:52.309 DEBUG:teuthology.orchestra.run.smithi145:> rm -rf -- /home/ubuntu/cephtest/archive
2023-12-26T18:17:52.311 DEBUG:teuthology.orchestra.run.smithi159:> rm -rf -- /home/ubuntu/cephtest/archive
2023-12-26T18:17:52.313 DEBUG:teuthology.orchestra.run.smithi183:> rm -rf -- /home/ubuntu/cephtest/archive
2023-12-26T18:17:52.358 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload
2023-12-26T18:17:52.368 INFO:teuthology.task.internal:Not uploading archives.
2023-12-26T18:17:52.369 DEBUG:teuthology.run_tasks:Unwinding manager internal.base
2023-12-26T18:17:52.387 INFO:teuthology.task.internal:Tidying up after the test...
2023-12-26T18:17:52.388 DEBUG:teuthology.orchestra.run.smithi145:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest
2023-12-26T18:17:52.390 DEBUG:teuthology.orchestra.run.smithi159:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest
2023-12-26T18:17:52.392 DEBUG:teuthology.orchestra.run.smithi183:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest
2023-12-26T18:17:52.402 INFO:teuthology.orchestra.run.smithi145.stdout:   262897      4 drwxr-xr-x   2  ubuntu   ubuntu       4096 Dec 26 18:17 /home/ubuntu/cephtest
2023-12-26T18:17:52.403 INFO:teuthology.orchestra.run.smithi159.stdout:   262183      4 drwxr-xr-x   2  ubuntu   ubuntu       4096 Dec 26 18:17 /home/ubuntu/cephtest
2023-12-26T18:17:52.412 INFO:teuthology.orchestra.run.smithi183.stdout:   263007      4 drwxr-xr-x   2  ubuntu   ubuntu       4096 Dec 26 18:17 /home/ubuntu/cephtest
2023-12-26T18:17:52.414 DEBUG:teuthology.run_tasks:Unwinding manager kernel
2023-12-26T18:17:52.423 DEBUG:teuthology.run_tasks:Unwinding manager console_log
2023-12-26T18:17:52.487 INFO:teuthology.nuke:Checking targets against current locks
2023-12-26T18:17:52.514 DEBUG:teuthology.nuke:shortname: smithi145
2023-12-26T18:17:52.514 INFO:teuthology.task.internal.check_lock:Checking locks...
2023-12-26T18:17:52.538 DEBUG:teuthology.nuke:shortname: smithi159
2023-12-26T18:17:52.538 INFO:teuthology.task.internal.check_lock:Checking locks...
2023-12-26T18:17:52.543 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi145.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi/7501638', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.4', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-12-26 18:04:59.566298', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFFIfNkyXHimgqOvMklW1zmsbUFLzqBd2H1r1kjoQpD8oKkpqkvq6N69HShfxzrjRI+gMAA9jdmCuGhNwCqWCzk='}
2023-12-26T18:17:52.565 DEBUG:teuthology.nuke:shortname: smithi183
2023-12-26T18:17:52.565 INFO:teuthology.task.internal.check_lock:Checking locks...
2023-12-26T18:17:52.568 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi159.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi/7501638', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.4', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-12-26 18:04:59.564572', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG8OL7Gf4ULcwMfSrr0LGQ0mNJVy53yuJWqdfMnOeRRej74j7gq1C+dnUdsCEZq/6BOM6rqMbeudFqsWJTcQPX0='}
2023-12-26T18:17:52.587 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi183.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-12-26_16:45:04-rbd-wip-yuri7-testing-2023-12-20-0808-pacific-distro-default-smithi/7501638', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.4', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-12-26 18:04:59.567816', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM2rv4UQndnVu33kHwptvszMFUQBTzzlU3nV8drYSf58Mudkatvf3KSkjsStMdm0tErpbYuR5/u3v9Jvhv7hATQ='}
2023-12-26T18:17:52.596 INFO:teuthology.orchestra.console.smithi145:Power off
2023-12-26T18:17:52.597 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2023-12-26T18:17:52.620 DEBUG:teuthology.orchestra.console.smithi145:power off output: Chassis Power Control: Down/Off
2023-12-26T18:17:52.620 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2023-12-26T18:17:52.641 INFO:teuthology.orchestra.console.smithi159:Power off
2023-12-26T18:17:52.642 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2023-12-26T18:17:52.652 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is on
2023-12-26T18:17:52.665 DEBUG:teuthology.orchestra.console.smithi159:power off output: Chassis Power Control: Down/Off
2023-12-26T18:17:52.665 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2023-12-26T18:17:52.680 INFO:teuthology.orchestra.console.smithi183:Power off
2023-12-26T18:17:52.681 DEBUG:teuthology.orchestra.console.smithi183:pexpect command: ipmitool -H smithi183.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2023-12-26T18:17:52.691 DEBUG:teuthology.orchestra.console.smithi159:check power output: Chassis Power is on
2023-12-26T18:17:52.704 DEBUG:teuthology.orchestra.console.smithi183:power off output: Chassis Power Control: Down/Off
2023-12-26T18:17:52.704 DEBUG:teuthology.orchestra.console.smithi183:pexpect command: ipmitool -H smithi183.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2023-12-26T18:17:52.728 DEBUG:teuthology.orchestra.console.smithi183:check power output: Chassis Power is on
2023-12-26T18:17:56.653 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2023-12-26T18:17:56.692 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2023-12-26T18:17:56.730 DEBUG:teuthology.orchestra.console.smithi183:pexpect command: ipmitool -H smithi183.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2023-12-26T18:17:56.764 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is on
2023-12-26T18:17:56.802 DEBUG:teuthology.orchestra.console.smithi159:check power output: Chassis Power is on
2023-12-26T18:17:56.840 DEBUG:teuthology.orchestra.console.smithi183:check power output: Chassis Power is on
2023-12-26T18:18:00.769 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2023-12-26T18:18:00.803 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2023-12-26T18:18:00.841 DEBUG:teuthology.orchestra.console.smithi183:pexpect command: ipmitool -H smithi183.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2023-12-26T18:18:00.881 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is on
2023-12-26T18:18:00.914 DEBUG:teuthology.orchestra.console.smithi159:check power output: Chassis Power is on
2023-12-26T18:18:00.952 DEBUG:teuthology.orchestra.console.smithi183:check power output: Chassis Power is off
2023-12-26T18:18:01.053 INFO:teuthology.orchestra.console.smithi183:Power off completed
2023-12-26T18:18:04.885 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2023-12-26T18:18:04.915 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2023-12-26T18:18:04.996 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is off
2023-12-26T18:18:05.025 DEBUG:teuthology.orchestra.console.smithi159:check power output: Chassis Power is off
2023-12-26T18:18:05.098 INFO:teuthology.orchestra.console.smithi145:Power off completed
2023-12-26T18:18:05.126 INFO:teuthology.orchestra.console.smithi159:Power off completed
2023-12-26T18:18:05.237 INFO:teuthology.run:Summary data:
description: rbd/migration/{1-base/install 2-clusters/{fixed-3 openstack} 3-objectstore/bluestore-hybrid
  4-supported-random-distro$/{rhel_8} 5-pool/ec-data-pool 6-prepare/qcow2-file 7-io-workloads/qemu_xfstests
  8-migrate-workloads/execute 9-cleanup/cleanup}
duration: 326.6231279373169
failure_reason: 'smithi159.front.sepia.ceph.com:  _ansible_no_log: false  changed:
  false  invocation:    module_args:      allow_downgrade: false      allowerasing:
  false      autoremove: false      bugfix: false      conf_file: null      disable_excludes:
  null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir:
  null      download_only: false      enable_plugin: []      enablerepo: []      exclude:
  []      install_repoquery: true      install_weak_deps: true      installroot: /      list:
  null      lock_timeout: 30      name:      - krb5-workstation      releasever: null      security:
  false      skip_broken: false      state: present      update_cache: false      update_only:
  false      validate_certs: true  msg: ''Failed to download metadata for repo ''''codeready-builder-for-rhel-8-x86_64-rpms'''':    Cannot
  download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were    tried''  rc:
  1  results: []smithi183.front.sepia.ceph.com:  _ansible_no_log: false  changed:
  false  invocation:    module_args:      allow_downgrade: false      allowerasing:
  false      autoremove: false      bugfix: false      conf_file: null      disable_excludes:
  null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir:
  null      download_only: false      enable_plugin: []      enablerepo: []      exclude:
  []      install_repoquery: true      install_weak_deps: true      installroot: /      list:
  null      lock_timeout: 30      name:      - krb5-workstation      releasever: null      security:
  false      skip_broken: false      state: present      update_cache: false      update_only:
  false      validate_certs: true  msg: ''Failed to download metadata for repo ''''codeready-builder-for-rhel-8-x86_64-rpms'''':    Cannot
  download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were    tried''  rc:
  1  results: []smithi145.front.sepia.ceph.com:  _ansible_no_log: false  changed:
  false  invocation:    module_args:      allow_downgrade: false      allowerasing:
  false      autoremove: false      bugfix: false      conf_file: null      disable_excludes:
  null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir:
  null      download_only: false      enable_plugin: []      enablerepo: []      exclude:
  []      install_repoquery: true      install_weak_deps: true      installroot: /      list:
  null      lock_timeout: 30      name:      - krb5-workstation      releasever: null      security:
  false      skip_broken: false      state: present      update_cache: false      update_only:
  false      validate_certs: true  msg: ''Failed to download metadata for repo ''''codeready-builder-for-rhel-8-x86_64-rpms'''':    Cannot
  download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were    tried''  rc:
  1  results: []'
owner: scheduled_yuriw@teuthology
sentry_event: https://sentry.ceph.com/organizations/ceph/?query=4c7831b67f894189881398d5f3dae705
status: dead
success: false

2023-12-26T18:18:05.237 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/
2023-12-26T18:18:05.323 INFO:teuthology.run:DEAD