2024-04-19T13:30:43.766 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d
2024-04-19T13:30:43.767 DEBUG:teuthology.run:Teuthology command: teuthology --name adking-2024-04-19_13:19:23-orch:cephadm-wip-adk-testing-2024-04-17-1713-distro-default-smithi --description orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc 0-nvme-loop 1-start 2-services/nfs-ingress2 3-final} --archive /home/teuthworker/archive/adking-2024-04-19_13:19:23-orch:cephadm-wip-adk-testing-2024-04-17-1713-distro-default-smithi/7663451 --verbose --owner scheduled_adking@teuthology -- /home/teuthworker/archive/adking-2024-04-19_13:19:23-orch:cephadm-wip-adk-testing-2024-04-17-1713-distro-default-smithi/7663451/orig.config.yaml
2024-04-19T13:30:43.803 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/
2024-04-19T13:30:43.882 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/adking-2024-04-19_13:19:23-orch:cephadm-wip-adk-testing-2024-04-17-1713-distro-default-smithi/7663451
  branch: wip-adk-testing-2024-04-17-1713
  description: orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc 0-nvme-loop
    1-start 2-services/nfs-ingress2 3-final}
  email: adking@redhat.com
  first_in_suite: false
  job_id: '7663451'
  kernel:
    kdb: 1
    sha1: distro
  last_in_suite: false
  machine_type: smithi
  name: adking-2024-04-19_13:19:23-orch:cephadm-wip-adk-testing-2024-04-17-1713-distro-default-smithi
  no_nested_subset: false
  nuke-on-error: true
  openstack:
  - volumes:
      count: 4
      size: 10
  os_type: centos
  os_version: 9.stream
  overrides:
    admin_socket:
      branch: wip-adk-testing-2024-04-17-1713
    ceph:
      conf:
        mgr:
          debug mgr: 20
          debug ms: 1
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
        osd:
          debug ms: 1
          debug osd: 20
          osd shutdown pgref assert: true
      flavor: default
      log-ignorelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      log-only-match:
      - CEPHADM_
      sha1: c53d131d890ca258dc8708b5590933577e6ceba0
    ceph-deploy:
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon: {}
    install:
      ceph:
        flavor: default
        sha1: c53d131d890ca258dc8708b5590933577e6ceba0
    selinux:
      allowlist:
      - scontext=system_u:system_r:logrotate_t:s0
    workunit:
      branch: wip-adk-testing-2024-04-17-1713
      sha1: c53d131d890ca258dc8708b5590933577e6ceba0
  owner: scheduled_adking@teuthology
  priority: 80
  repo: https://git.ceph.com/ceph-ci.git
  roles:
  - - host.a
    - client.0
  - - host.b
    - client.1
  seed: 7132
  sha1: c53d131d890ca258dc8708b5590933577e6ceba0
  sleep_before_teardown: 0
  subset: 1/16
  suite: orch:cephadm
  suite_branch: wip-adk-testing-2024-04-17-1713
  suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_c53d131d890ca258dc8708b5590933577e6ceba0/qa
  suite_relpath: qa
  suite_repo: https://git.ceph.com/ceph-ci.git
  suite_sha1: c53d131d890ca258dc8708b5590933577e6ceba0
  targets:
    smithi145.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMFEFBaZCyv0FZIYfZt1oWSZLs8KGDjkh5OlrfdBKZN6JkPcE1aXaBstW48vAZOvwvcU6ra46GAlaM9dfXx/B+M=
    smithi167.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHopHKQ3L0HXp/0IHScTj04oJEJcsVCW8FOFOuoUVmGC0a9R31+15ldw0ZDRmSV0YjhRkug6BwogL0mxOQMPXgQ=
  tasks:
  - pexec:
      all:
      - sudo dnf install runc -y
      - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf
      - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf
  - nvme_loop: null
  - cephadm:
      roleless: true
  - cephadm.shell:
      host.a:
      - ceph orch status
      - ceph orch ps
      - ceph orch ls
      - ceph orch host ls
      - ceph orch device ls
  - vip: null
  - cephadm.shell:
      host.a:
      - ceph orch device ls --refresh
  - vip.exec:
      all-hosts:
      - systemctl stop nfs-server
  - cephadm.shell:
      host.a:
      - ceph fs volume create foofs
      - ceph nfs cluster create foo --ingress --virtual-ip {{VIP0}}/{{VIPPREFIXLEN}}
        --port 2999
      - ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path
        /fake
  - cephadm.wait_for_service:
      service: nfs.foo
  - cephadm.wait_for_service:
      service: ingress.nfs.foo
  - vip.exec:
      host.a:
      - mkdir /mnt/foo
      - sleep 5
      - mount -t nfs {{VIP0}}:/fake /mnt/foo -o port=2999
      - echo test > /mnt/foo/testfile
      - sync
  - cephadm.shell:
      host.a:
      - "echo \"Check with each haproxy down in turn...\"\nfor haproxy in `ceph orch\
        \ ps | grep ^haproxy.nfs.foo. | awk '{print $1}'`; do\n  ceph orch daemon stop\
        \ $haproxy\n  while ! ceph orch ps | grep $haproxy | grep stopped; do sleep\
        \ 1 ; done\n  cat /mnt/foo/testfile\n  echo $haproxy > /mnt/foo/testfile\n \
        \ sync\n  ceph orch daemon start $haproxy\n  while ! ceph orch ps | grep $haproxy\
        \ | grep running; do sleep 1 ; done\ndone\n"
      volumes:
      - /mnt/foo:/mnt/foo
  - vip.exec:
      all-hosts:
      - "echo \"Check with $(hostname) ganesha(s) down...\"\nfor c in `systemctl | grep\
        \ ceph- | grep @nfs | awk '{print $1}'`; do\n    cid=`echo $c | sed 's/@/-/'`\n\
        \    id=`echo $c | cut -d @ -f 2 | sed 's/.service$//'`\n    fsid=`echo $c |\
        \ cut -d @ -f 1 | cut -d - -f 2-`\n    echo \"Removing daemon $id fsid $fsid...\"\
        \n    sudo $TESTDIR/cephadm rm-daemon --fsid $fsid --name $id\n\n    echo \"\
        Waking up cephadm...\"\n    sudo $TESTDIR/cephadm shell -- ceph orch ps --refresh\n\
        \n    while ! timeout 1 cat /mnt/foo/testfile ; do true ; done\n    echo \"\
        Mount is back!\"\ndone\n"
  - cephadm.shell:
      host.a:
      - stat -c '%u %g' /var/log/ceph | grep '167 167'
      - ceph orch status
      - ceph orch ps
      - ceph orch ls
      - ceph orch host ls
      - ceph orch device ls
      - ceph orch ls | grep '^osd.all-available-devices '
  teuthology:
    fragments_dropped: []
    meta: {}
    postmerge: []
  teuthology_branch: main
  teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17
  timestamp: 2024-04-19_13:19:23
  tube: smithi
  user: adking
  verbose: false
  worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885
2024-04-19T13:30:43.882 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_c53d131d890ca258dc8708b5590933577e6ceba0/qa; will attempt to use it
2024-04-19T13:30:43.883 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_c53d131d890ca258dc8708b5590933577e6ceba0/qa/tasks
2024-04-19T13:30:43.883 INFO:teuthology.run_tasks:Running task internal.check_packages...
2024-04-19T13:30:43.885 INFO:teuthology.task.internal:Checking packages...
2024-04-19T13:30:43.908 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'c53d131d890ca258dc8708b5590933577e6ceba0'
2024-04-19T13:30:43.908 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2024-04-19T13:30:43.908 INFO:teuthology.packaging:ref: None
2024-04-19T13:30:43.908 INFO:teuthology.packaging:tag: None
2024-04-19T13:30:43.908 INFO:teuthology.packaging:branch: wip-adk-testing-2024-04-17-1713
2024-04-19T13:30:43.908 INFO:teuthology.packaging:sha1: c53d131d890ca258dc8708b5590933577e6ceba0
2024-04-19T13:30:43.909 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk-testing-2024-04-17-1713
2024-04-19T13:30:44.223 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3175.gc53d131d
2024-04-19T13:30:44.224 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2024-04-19T13:30:44.230 INFO:teuthology.task.internal:no buildpackages task found
2024-04-19T13:30:44.230 INFO:teuthology.run_tasks:Running task internal.save_config...
2024-04-19T13:30:44.260 INFO:teuthology.task.internal:Saving configuration
2024-04-19T13:30:44.274 INFO:teuthology.run_tasks:Running task internal.check_lock...
2024-04-19T13:30:44.302 INFO:teuthology.task.internal.check_lock:Checking locks...
2024-04-19T13:30:44.319 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi145.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-19_13:19:23-orch:cephadm-wip-adk-testing-2024-04-17-1713-distro-default-smithi/7663451', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-19 13:20:48.544443', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMFEFBaZCyv0FZIYfZt1oWSZLs8KGDjkh5OlrfdBKZN6JkPcE1aXaBstW48vAZOvwvcU6ra46GAlaM9dfXx/B+M='}
2024-04-19T13:30:44.339 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi167.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-19_13:19:23-orch:cephadm-wip-adk-testing-2024-04-17-1713-distro-default-smithi/7663451', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-19 13:20:48.545681', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHopHKQ3L0HXp/0IHScTj04oJEJcsVCW8FOFOuoUVmGC0a9R31+15ldw0ZDRmSV0YjhRkug6BwogL0mxOQMPXgQ='}
2024-04-19T13:30:44.339 INFO:teuthology.run_tasks:Running task internal.add_remotes...
2024-04-19T13:30:44.345 INFO:teuthology.task.internal:roles: ubuntu@smithi145.front.sepia.ceph.com - ['host.a', 'client.0']
2024-04-19T13:30:44.345 INFO:teuthology.task.internal:roles: ubuntu@smithi167.front.sepia.ceph.com - ['host.b', 'client.1']
2024-04-19T13:30:44.345 INFO:teuthology.run_tasks:Running task console_log...
2024-04-19T13:30:44.435 DEBUG:teuthology.exit:Installing handler: Handler(exiter=<teuthology.exit.Exiter object at 0x7f469b6c8790>, func=<function ConsoleLog.start_logging.<locals>.kill_console_loggers at 0x7f469c4b6160>, signals=[15])
2024-04-19T13:30:44.436 INFO:teuthology.run_tasks:Running task internal.connect...
2024-04-19T13:30:44.442 INFO:teuthology.task.internal:Opening connections...
2024-04-19T13:30:44.442 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi145.front.sepia.ceph.com
2024-04-19T13:30:44.445 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T13:30:44.539 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi167.front.sepia.ceph.com
2024-04-19T13:30:44.540 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T13:30:44.626 INFO:teuthology.run_tasks:Running task internal.push_inventory...
2024-04-19T13:30:44.633 DEBUG:teuthology.orchestra.run.smithi145:> uname -m
2024-04-19T13:30:44.651 INFO:teuthology.orchestra.run.smithi145.stdout:x86_64
2024-04-19T13:30:44.651 DEBUG:teuthology.orchestra.run.smithi145:> cat /etc/os-release
2024-04-19T13:30:44.709 INFO:teuthology.orchestra.run.smithi145.stdout:NAME="CentOS Stream"
2024-04-19T13:30:44.709 INFO:teuthology.orchestra.run.smithi145.stdout:VERSION="9"
2024-04-19T13:30:44.709 INFO:teuthology.orchestra.run.smithi145.stdout:ID="centos"
2024-04-19T13:30:44.709 INFO:teuthology.orchestra.run.smithi145.stdout:ID_LIKE="rhel fedora"
2024-04-19T13:30:44.709 INFO:teuthology.orchestra.run.smithi145.stdout:VERSION_ID="9"
2024-04-19T13:30:44.709 INFO:teuthology.orchestra.run.smithi145.stdout:PLATFORM_ID="platform:el9"
2024-04-19T13:30:44.709 INFO:teuthology.orchestra.run.smithi145.stdout:PRETTY_NAME="CentOS Stream 9"
2024-04-19T13:30:44.709 INFO:teuthology.orchestra.run.smithi145.stdout:ANSI_COLOR="0;31"
2024-04-19T13:30:44.709 INFO:teuthology.orchestra.run.smithi145.stdout:LOGO="fedora-logo-icon"
2024-04-19T13:30:44.709 INFO:teuthology.orchestra.run.smithi145.stdout:CPE_NAME="cpe:/o:centos:centos:9"
2024-04-19T13:30:44.709 INFO:teuthology.orchestra.run.smithi145.stdout:HOME_URL="https://centos.org/"
2024-04-19T13:30:44.709 INFO:teuthology.orchestra.run.smithi145.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/"
2024-04-19T13:30:44.710 INFO:teuthology.orchestra.run.smithi145.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9"
2024-04-19T13:30:44.710 INFO:teuthology.orchestra.run.smithi145.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream"
2024-04-19T13:30:44.710 INFO:teuthology.lock.ops:Updating smithi145.front.sepia.ceph.com on lock server
2024-04-19T13:30:44.732 DEBUG:teuthology.orchestra.run.smithi167:> uname -m
2024-04-19T13:30:44.751 INFO:teuthology.orchestra.run.smithi167.stdout:x86_64
2024-04-19T13:30:44.751 DEBUG:teuthology.orchestra.run.smithi167:> cat /etc/os-release
2024-04-19T13:30:44.808 INFO:teuthology.orchestra.run.smithi167.stdout:NAME="CentOS Stream"
2024-04-19T13:30:44.809 INFO:teuthology.orchestra.run.smithi167.stdout:VERSION="9"
2024-04-19T13:30:44.809 INFO:teuthology.orchestra.run.smithi167.stdout:ID="centos"
2024-04-19T13:30:44.809 INFO:teuthology.orchestra.run.smithi167.stdout:ID_LIKE="rhel fedora"
2024-04-19T13:30:44.809 INFO:teuthology.orchestra.run.smithi167.stdout:VERSION_ID="9"
2024-04-19T13:30:44.809 INFO:teuthology.orchestra.run.smithi167.stdout:PLATFORM_ID="platform:el9"
2024-04-19T13:30:44.809 INFO:teuthology.orchestra.run.smithi167.stdout:PRETTY_NAME="CentOS Stream 9"
2024-04-19T13:30:44.809 INFO:teuthology.orchestra.run.smithi167.stdout:ANSI_COLOR="0;31"
2024-04-19T13:30:44.809 INFO:teuthology.orchestra.run.smithi167.stdout:LOGO="fedora-logo-icon"
2024-04-19T13:30:44.809 INFO:teuthology.orchestra.run.smithi167.stdout:CPE_NAME="cpe:/o:centos:centos:9"
2024-04-19T13:30:44.809 INFO:teuthology.orchestra.run.smithi167.stdout:HOME_URL="https://centos.org/"
2024-04-19T13:30:44.809 INFO:teuthology.orchestra.run.smithi167.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/"
2024-04-19T13:30:44.809 INFO:teuthology.orchestra.run.smithi167.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9"
2024-04-19T13:30:44.810 INFO:teuthology.orchestra.run.smithi167.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream"
2024-04-19T13:30:44.810 INFO:teuthology.lock.ops:Updating smithi167.front.sepia.ceph.com on lock server
2024-04-19T13:30:44.832 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles...
2024-04-19T13:30:44.841 INFO:teuthology.run_tasks:Running task internal.check_conflict...
2024-04-19T13:30:44.870 INFO:teuthology.task.internal:Checking for old test directory...
2024-04-19T13:30:44.871 DEBUG:teuthology.orchestra.run.smithi145:> test '!' -e /home/ubuntu/cephtest
2024-04-19T13:30:44.874 DEBUG:teuthology.orchestra.run.smithi167:> test '!' -e /home/ubuntu/cephtest
2024-04-19T13:30:44.889 INFO:teuthology.run_tasks:Running task internal.check_ceph_data...
2024-04-19T13:30:44.895 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph...
2024-04-19T13:30:44.896 DEBUG:teuthology.orchestra.run.smithi145:> test -z $(ls -A /var/lib/ceph)
2024-04-19T13:30:44.931 DEBUG:teuthology.orchestra.run.smithi167:> test -z $(ls -A /var/lib/ceph)
2024-04-19T13:30:44.973 INFO:teuthology.run_tasks:Running task internal.vm_setup...
2024-04-19T13:30:45.017 INFO:teuthology.run_tasks:Running task kernel...
2024-04-19T13:30:45.032 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'}
2024-04-19T13:30:45.033 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300
2024-04-19T13:30:45.033 DEBUG:teuthology.orchestra.run.smithi145:> test -f /run/.containerenv -o -f /.dockerenv
2024-04-19T13:30:45.033 DEBUG:teuthology.orchestra.run.smithi167:> test -f /run/.containerenv -o -f /.dockerenv
2024-04-19T13:30:45.049 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T13:30:45.050 DEBUG:teuthology.orchestra.run.smithi145:> uname -r
2024-04-19T13:30:45.051 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T13:30:45.051 DEBUG:teuthology.orchestra.run.smithi167:> uname -r
2024-04-19T13:30:45.107 INFO:teuthology.orchestra.run.smithi145.stdout:5.14.0-361.el9.x86_64
2024-04-19T13:30:45.107 INFO:teuthology.task.kernel:Running kernel on smithi145: 5.14.0-361.el9.x86_64
2024-04-19T13:30:45.108 DEBUG:teuthology.orchestra.run.smithi145:> sudo yum install -y kernel
2024-04-19T13:30:45.109 INFO:teuthology.orchestra.run.smithi167.stdout:5.14.0-361.el9.x86_64
2024-04-19T13:30:45.109 INFO:teuthology.task.kernel:Running kernel on smithi167: 5.14.0-361.el9.x86_64
2024-04-19T13:30:45.109 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum install -y kernel
2024-04-19T13:30:47.508 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS-9 - AppStream                            8.8 MB/s | 6.4 MB     00:00
2024-04-19T13:30:47.705 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS-9 - AppStream                            7.1 MB/s | 6.4 MB     00:00
2024-04-19T13:30:49.134 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS-9 - BaseOS                               3.9 MB/s | 2.0 MB     00:00
2024-04-19T13:30:49.180 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS-9 - BaseOS                               5.3 MB/s | 2.0 MB     00:00
2024-04-19T13:30:50.107 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 9 - BaseOS                         13 MB/s | 8.0 MB     00:00
2024-04-19T13:30:50.738 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - BaseOS                        6.5 MB/s | 8.0 MB     00:01
2024-04-19T13:30:53.399 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - AppStream                      12 MB/s |  19 MB     00:01
2024-04-19T13:30:55.858 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 9 - AppStream                     3.9 MB/s |  19 MB     00:04
2024-04-19T13:30:57.564 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - CRB                            11 MB/s | 6.2 MB     00:00
2024-04-19T13:30:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - Extras packages                30 kB/s |  16 kB     00:00
2024-04-19T13:31:01.184 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 9 - CRB                           3.5 MB/s | 6.2 MB     00:01
2024-04-19T13:31:02.710 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 9 - Extras packages                45 kB/s |  16 kB     00:00
2024-04-19T13:31:03.703 INFO:teuthology.orchestra.run.smithi167.stdout:Extra Packages for Enterprise Linux              25 MB/s |  21 MB     00:00
2024-04-19T13:31:05.043 INFO:teuthology.orchestra.run.smithi145.stdout:Extra Packages for Enterprise Linux             3.7 MB/s |  21 MB     00:05
2024-04-19T13:31:08.844 INFO:teuthology.orchestra.run.smithi167.stdout:lab-extras                                       16 kB/s | 1.7 kB     00:00
2024-04-19T13:31:10.357 INFO:teuthology.orchestra.run.smithi145.stdout:lab-extras                                      5.0 kB/s | 1.7 kB     00:00
2024-04-19T13:31:11.310 INFO:teuthology.orchestra.run.smithi167.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed.
2024-04-19T13:31:11.310 INFO:teuthology.orchestra.run.smithi167.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed.
2024-04-19T13:31:11.397 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved.
2024-04-19T13:31:11.406 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================
2024-04-19T13:31:11.406 INFO:teuthology.orchestra.run.smithi167.stdout: Package                Arch      Version                Repository        Size
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout:Installing:
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout: kernel                 x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     29 k
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies:
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout: kernel-core            x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     16 M
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout: kernel-modules         x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     34 M
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout: kernel-modules-core    x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     27 M
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout:
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout:Install  4 Packages
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout:
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 78 M
2024-04-19T13:31:11.407 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 125 M
2024-04-19T13:31:11.408 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages:
2024-04-19T13:31:11.612 INFO:teuthology.orchestra.run.smithi167.stdout:(1/4): kernel-6.9.0-0.rc2.1.el10.x86_64.rpm     140 kB/s |  29 kB     00:00
2024-04-19T13:31:12.834 INFO:teuthology.orchestra.run.smithi145.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed.
2024-04-19T13:31:12.834 INFO:teuthology.orchestra.run.smithi145.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed.
2024-04-19T13:31:12.921 INFO:teuthology.orchestra.run.smithi145.stdout:Dependencies resolved.
2024-04-19T13:31:12.929 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================
2024-04-19T13:31:12.930 INFO:teuthology.orchestra.run.smithi145.stdout: Package                Arch      Version                Repository        Size
2024-04-19T13:31:12.930 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================
2024-04-19T13:31:12.930 INFO:teuthology.orchestra.run.smithi145.stdout:Installing:
2024-04-19T13:31:12.930 INFO:teuthology.orchestra.run.smithi145.stdout: kernel                 x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     29 k
2024-04-19T13:31:12.930 INFO:teuthology.orchestra.run.smithi145.stdout:Installing dependencies:
2024-04-19T13:31:12.930 INFO:teuthology.orchestra.run.smithi145.stdout: kernel-core            x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     16 M
2024-04-19T13:31:12.930 INFO:teuthology.orchestra.run.smithi145.stdout: kernel-modules         x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     34 M
2024-04-19T13:31:12.930 INFO:teuthology.orchestra.run.smithi145.stdout: kernel-modules-core    x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     27 M
2024-04-19T13:31:12.930 INFO:teuthology.orchestra.run.smithi145.stdout:
2024-04-19T13:31:12.930 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction Summary
2024-04-19T13:31:12.930 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================
2024-04-19T13:31:12.931 INFO:teuthology.orchestra.run.smithi145.stdout:Install  4 Packages
2024-04-19T13:31:12.931 INFO:teuthology.orchestra.run.smithi145.stdout:
2024-04-19T13:31:12.931 INFO:teuthology.orchestra.run.smithi145.stdout:Total download size: 78 M
2024-04-19T13:31:12.931 INFO:teuthology.orchestra.run.smithi145.stdout:Installed size: 125 M
2024-04-19T13:31:12.931 INFO:teuthology.orchestra.run.smithi145.stdout:Downloading Packages:
2024-04-19T13:31:13.088 INFO:teuthology.orchestra.run.smithi167.stdout:(2/4): kernel-core-6.9.0-0.rc2.1.el10.x86_64.rp 9.6 MB/s |  16 MB     00:01
2024-04-19T13:31:13.158 INFO:teuthology.orchestra.run.smithi145.stdout:(1/4): kernel-6.9.0-0.rc2.1.el10.x86_64.rpm     126 kB/s |  29 kB     00:00
2024-04-19T13:31:13.379 INFO:teuthology.orchestra.run.smithi167.stdout:(3/4): kernel-modules-core-6.9.0-0.rc2.1.el10.x  15 MB/s |  27 MB     00:01
2024-04-19T13:31:14.421 INFO:teuthology.orchestra.run.smithi167.stdout:(4/4): kernel-modules-6.9.0-0.rc2.1.el10.x86_64  11 MB/s |  34 MB     00:03
2024-04-19T13:31:14.424 INFO:teuthology.orchestra.run.smithi167.stdout:--------------------------------------------------------------------------------
2024-04-19T13:31:14.424 INFO:teuthology.orchestra.run.smithi167.stdout:Total                                            26 MB/s |  78 MB     00:03
2024-04-19T13:31:14.425 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check
2024-04-19T13:31:14.811 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded.
2024-04-19T13:31:14.811 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test
2024-04-19T13:31:15.641 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded.
2024-04-19T13:31:15.641 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction
2024-04-19T13:31:16.349 INFO:teuthology.orchestra.run.smithi145.stdout:(2/4): kernel-modules-core-6.9.0-0.rc2.1.el10.x 8.6 MB/s |  27 MB     00:03
2024-04-19T13:31:16.541 INFO:teuthology.orchestra.run.smithi145.stdout:(3/4): kernel-core-6.9.0-0.rc2.1.el10.x86_64.rp 4.5 MB/s |  16 MB     00:03
2024-04-19T13:31:17.008 INFO:teuthology.orchestra.run.smithi145.stdout:(4/4): kernel-modules-6.9.0-0.rc2.1.el10.x86_64 8.4 MB/s |  34 MB     00:04
2024-04-19T13:31:17.011 INFO:teuthology.orchestra.run.smithi145.stdout:--------------------------------------------------------------------------------
2024-04-19T13:31:17.011 INFO:teuthology.orchestra.run.smithi145.stdout:Total                                            19 MB/s |  78 MB     00:04
2024-04-19T13:31:17.012 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction check
2024-04-19T13:31:17.380 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction check succeeded.
2024-04-19T13:31:17.380 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction test
2024-04-19T13:31:17.899 INFO:teuthology.orchestra.run.smithi167.stdout:  Preparing        :                                                        1/1
2024-04-19T13:31:18.223 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction test succeeded.
2024-04-19T13:31:18.223 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction
2024-04-19T13:31:18.724 INFO:teuthology.orchestra.run.smithi167.stdout:  Installing       : kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64          1/4
2024-04-19T13:31:18.814 INFO:teuthology.orchestra.run.smithi167.stdout:  Installing       : kernel-core-6.9.0-0.rc2.1.el10.x86_64                  2/4
2024-04-19T13:31:20.427 INFO:teuthology.orchestra.run.smithi145.stdout:  Preparing        :                                                        1/1
2024-04-19T13:31:21.086 INFO:teuthology.orchestra.run.smithi145.stdout:  Installing       : kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64          1/4
2024-04-19T13:31:21.157 INFO:teuthology.orchestra.run.smithi167.stdout:  Running scriptlet: kernel-core-6.9.0-0.rc2.1.el10.x86_64                  2/4
2024-04-19T13:31:21.191 INFO:teuthology.orchestra.run.smithi145.stdout:  Installing       : kernel-core-6.9.0-0.rc2.1.el10.x86_64                  2/4
2024-04-19T13:31:21.296 INFO:teuthology.orchestra.run.smithi167.stdout:  Installing       : kernel-modules-6.9.0-0.rc2.1.el10.x86_64               3/4
2024-04-19T13:31:23.532 INFO:teuthology.orchestra.run.smithi145.stdout:  Running scriptlet: kernel-core-6.9.0-0.rc2.1.el10.x86_64                  2/4
2024-04-19T13:31:23.664 INFO:teuthology.orchestra.run.smithi145.stdout:  Installing       : kernel-modules-6.9.0-0.rc2.1.el10.x86_64               3/4
2024-04-19T13:31:26.090 INFO:teuthology.orchestra.run.smithi167.stdout:  Running scriptlet: kernel-modules-6.9.0-0.rc2.1.el10.x86_64               3/4
2024-04-19T13:31:26.136 INFO:teuthology.orchestra.run.smithi167.stdout:  Installing       : kernel-6.9.0-0.rc2.1.el10.x86_64                       4/4
2024-04-19T13:31:28.434 INFO:teuthology.orchestra.run.smithi145.stdout:  Running scriptlet: kernel-modules-6.9.0-0.rc2.1.el10.x86_64               3/4
2024-04-19T13:31:28.538 INFO:teuthology.orchestra.run.smithi145.stdout:  Installing       : kernel-6.9.0-0.rc2.1.el10.x86_64                       4/4
2024-04-19T13:31:31.018 INFO:teuthology.orchestra.run.smithi167.stdout:  Running scriptlet: kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64          4/4
2024-04-19T13:31:33.349 INFO:teuthology.orchestra.run.smithi145.stdout:  Running scriptlet: kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64          4/4
2024-04-19T13:32:00.763 INFO:teuthology.orchestra.run.smithi167.stdout:  Running scriptlet: kernel-core-6.9.0-0.rc2.1.el10.x86_64                  4/4
2024-04-19T13:32:00.763 INFO:teuthology.orchestra.run.smithi167.stdout:
2024-04-19T13:32:00.763 INFO:teuthology.orchestra.run.smithi167.stdout:gzip: /lib/modules/6.9.0-0.rc2.1.el10.x86_64/symvers.xz: not in gzip format
2024-04-19T13:32:00.763 INFO:teuthology.orchestra.run.smithi167.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y
2024-04-19T13:32:00.763 INFO:teuthology.orchestra.run.smithi167.stdout:
2024-04-19T13:32:00.807 INFO:teuthology.orchestra.run.smithi167.stdout:  Running scriptlet: kernel-modules-6.9.0-0.rc2.1.el10.x86_64               4/4
2024-04-19T13:32:01.198 INFO:teuthology.orchestra.run.smithi167.stdout:  Running scriptlet: kernel-6.9.0-0.rc2.1.el10.x86_64                       4/4
2024-04-19T13:32:01.198 INFO:teuthology.orchestra.run.smithi167.stdout:  Verifying        : kernel-6.9.0-0.rc2.1.el10.x86_64                       1/4
2024-04-19T13:32:01.198 INFO:teuthology.orchestra.run.smithi167.stdout:  Verifying        : kernel-core-6.9.0-0.rc2.1.el10.x86_64                  2/4
2024-04-19T13:32:01.198 INFO:teuthology.orchestra.run.smithi167.stdout:  Verifying        : kernel-modules-6.9.0-0.rc2.1.el10.x86_64               3/4
2024-04-19T13:32:01.652 INFO:teuthology.orchestra.run.smithi167.stdout:  Verifying        : kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64          4/4
2024-04-19T13:32:01.652 INFO:teuthology.orchestra.run.smithi167.stdout:
2024-04-19T13:32:01.652 INFO:teuthology.orchestra.run.smithi167.stdout:Installed:
2024-04-19T13:32:01.652 INFO:teuthology.orchestra.run.smithi167.stdout:  kernel-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:01.652 INFO:teuthology.orchestra.run.smithi167.stdout:  kernel-core-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:01.652 INFO:teuthology.orchestra.run.smithi167.stdout:  kernel-modules-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:01.652 INFO:teuthology.orchestra.run.smithi167.stdout:  kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:01.652 INFO:teuthology.orchestra.run.smithi167.stdout:
2024-04-19T13:32:01.652 INFO:teuthology.orchestra.run.smithi167.stdout:Complete!
2024-04-19T13:32:01.974 DEBUG:teuthology.orchestra.run.smithi167:> rpm -q kernel | sort -rV | head -n 1
2024-04-19T13:32:02.086 INFO:teuthology.orchestra.run.smithi167.stdout:kernel-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:02.086 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:02.086 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:02.086 INFO:teuthology.task.kernel:Skipping firmware on distro kernel
2024-04-19T13:32:02.086 DEBUG:teuthology.task.kernel:src is distro, skipping download
2024-04-19T13:32:02.087 INFO:teuthology.task.kernel:Installing distro kernel on host.b...
2024-04-19T13:32:02.087 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi167.front.sepia.ceph.com, path=None, version=distro)
2024-04-19T13:32:02.087 DEBUG:teuthology.orchestra.run.smithi167:> rpm -q kernel | sort -rV | head -n 1
2024-04-19T13:32:02.166 INFO:teuthology.orchestra.run.smithi167.stdout:kernel-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:02.166 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:02.169 DEBUG:teuthology.orchestra.run.smithi167:> sudo rpm -qi grub2-tools
2024-04-19T13:32:02.203 INFO:teuthology.orchestra.run.smithi167.stdout:Name        : grub2-tools
2024-04-19T13:32:02.203 INFO:teuthology.orchestra.run.smithi167.stdout:Epoch       : 1
2024-04-19T13:32:02.203 INFO:teuthology.orchestra.run.smithi167.stdout:Version     : 2.06
2024-04-19T13:32:02.203 INFO:teuthology.orchestra.run.smithi167.stdout:Release     : 68.el9
2024-04-19T13:32:02.204 INFO:teuthology.orchestra.run.smithi167.stdout:Architecture: x86_64
2024-04-19T13:32:02.204 INFO:teuthology.orchestra.run.smithi167.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC
2024-04-19T13:32:02.204 INFO:teuthology.orchestra.run.smithi167.stdout:Group       : Unspecified
2024-04-19T13:32:02.204 INFO:teuthology.orchestra.run.smithi167.stdout:Size        : 8264254
2024-04-19T13:32:02.204 INFO:teuthology.orchestra.run.smithi167.stdout:License     : GPLv3+
2024-04-19T13:32:02.204 INFO:teuthology.orchestra.run.smithi167.stdout:Signature   : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d
2024-04-19T13:32:02.204 INFO:teuthology.orchestra.run.smithi167.stdout:Source RPM  : grub2-2.06-68.el9.src.rpm
2024-04-19T13:32:02.204 INFO:teuthology.orchestra.run.smithi167.stdout:Build Date  : Wed 02 Aug 2023 03:15:48 PM UTC
2024-04-19T13:32:02.204 INFO:teuthology.orchestra.run.smithi167.stdout:Build Host  : x86-05.stream.rdu2.redhat.com
2024-04-19T13:32:02.204 INFO:teuthology.orchestra.run.smithi167.stdout:Packager    : builder@centos.org
2024-04-19T13:32:02.204 INFO:teuthology.orchestra.run.smithi167.stdout:Vendor      : CentOS
2024-04-19T13:32:02.204 INFO:teuthology.orchestra.run.smithi167.stdout:URL         : http://www.gnu.org/software/grub/
2024-04-19T13:32:02.204 INFO:teuthology.orchestra.run.smithi167.stdout:Summary     : Support tools for GRUB.
2024-04-19T13:32:02.205 INFO:teuthology.orchestra.run.smithi167.stdout:Description :
2024-04-19T13:32:02.205 INFO:teuthology.orchestra.run.smithi167.stdout:
2024-04-19T13:32:02.205 INFO:teuthology.orchestra.run.smithi167.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and
2024-04-19T13:32:02.205 INFO:teuthology.orchestra.run.smithi167.stdout:customizable bootloader with modular architecture.  It supports a rich
2024-04-19T13:32:02.205 INFO:teuthology.orchestra.run.smithi167.stdout:variety of kernel formats, file systems, computer architectures and
2024-04-19T13:32:02.205 INFO:teuthology.orchestra.run.smithi167.stdout:hardware devices.
2024-04-19T13:32:02.205 INFO:teuthology.orchestra.run.smithi167.stdout:
2024-04-19T13:32:02.205 INFO:teuthology.orchestra.run.smithi167.stdout:This subpackage provides tools for support of all platforms.
2024-04-19T13:32:02.206 INFO:teuthology.task.kernel:Updating Grub Version: grub2
2024-04-19T13:32:02.206 INFO:teuthology.task.kernel:Updating grub on smithi167 to boot 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:02.206 DEBUG:teuthology.orchestra.run.smithi167:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg
2024-04-19T13:32:02.798 INFO:teuthology.orchestra.run.smithi167.stderr:Generating grub configuration file ...
2024-04-19T13:32:02.971 INFO:teuthology.orchestra.run.smithi145.stdout:  Running scriptlet: kernel-core-6.9.0-0.rc2.1.el10.x86_64                  4/4
2024-04-19T13:32:02.971 INFO:teuthology.orchestra.run.smithi145.stdout:
2024-04-19T13:32:02.971 INFO:teuthology.orchestra.run.smithi145.stdout:gzip: /lib/modules/6.9.0-0.rc2.1.el10.x86_64/symvers.xz: not in gzip format
2024-04-19T13:32:02.972 INFO:teuthology.orchestra.run.smithi145.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y
2024-04-19T13:32:02.972 INFO:teuthology.orchestra.run.smithi145.stdout:
2024-04-19T13:32:03.012 INFO:teuthology.orchestra.run.smithi145.stdout:  Running scriptlet: kernel-modules-6.9.0-0.rc2.1.el10.x86_64               4/4
2024-04-19T13:32:03.389 INFO:teuthology.orchestra.run.smithi145.stdout:  Running scriptlet: kernel-6.9.0-0.rc2.1.el10.x86_64                       4/4
2024-04-19T13:32:03.389 INFO:teuthology.orchestra.run.smithi145.stdout:  Verifying        : kernel-6.9.0-0.rc2.1.el10.x86_64                       1/4
2024-04-19T13:32:03.389 INFO:teuthology.orchestra.run.smithi145.stdout:  Verifying        : kernel-core-6.9.0-0.rc2.1.el10.x86_64                  2/4
2024-04-19T13:32:03.389 INFO:teuthology.orchestra.run.smithi145.stdout:  Verifying        : kernel-modules-6.9.0-0.rc2.1.el10.x86_64               3/4
2024-04-19T13:32:03.880 INFO:teuthology.orchestra.run.smithi145.stdout:  Verifying        : kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64          4/4
2024-04-19T13:32:03.880 INFO:teuthology.orchestra.run.smithi145.stdout:
2024-04-19T13:32:03.880 INFO:teuthology.orchestra.run.smithi145.stdout:Installed:
2024-04-19T13:32:03.880 INFO:teuthology.orchestra.run.smithi145.stdout:  kernel-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:03.880 INFO:teuthology.orchestra.run.smithi145.stdout:  kernel-core-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:03.880 INFO:teuthology.orchestra.run.smithi145.stdout:  kernel-modules-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:03.880 INFO:teuthology.orchestra.run.smithi145.stdout:  kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:03.880 INFO:teuthology.orchestra.run.smithi145.stdout:
2024-04-19T13:32:03.880 INFO:teuthology.orchestra.run.smithi145.stdout:Complete!
2024-04-19T13:32:04.208 DEBUG:teuthology.orchestra.run.smithi145:> rpm -q kernel | sort -rV | head -n 1
2024-04-19T13:32:04.217 INFO:teuthology.orchestra.run.smithi167.stderr:Adding boot menu entry for UEFI Firmware Settings ...
2024-04-19T13:32:04.262 INFO:teuthology.orchestra.run.smithi167.stderr:done
2024-04-19T13:32:04.264 DEBUG:teuthology.orchestra.run.smithi167:> mktemp
2024-04-19T13:32:04.279 INFO:teuthology.orchestra.run.smithi167.stdout:/tmp/tmp.LZlGHoHphz
2024-04-19T13:32:04.279 DEBUG:teuthology.orchestra.run.smithi167:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.LZlGHoHphz
2024-04-19T13:32:04.302 INFO:teuthology.orchestra.run.smithi145.stdout:kernel-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:04.302 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:04.302 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:04.302 INFO:teuthology.task.kernel:Skipping firmware on distro kernel
2024-04-19T13:32:04.303 DEBUG:teuthology.task.kernel:src is distro, skipping download
2024-04-19T13:32:04.303 INFO:teuthology.task.kernel:Installing distro kernel on host.a...
2024-04-19T13:32:04.303 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi145.front.sepia.ceph.com, path=None, version=distro)
2024-04-19T13:32:04.303 DEBUG:teuthology.orchestra.run.smithi145:> rpm -q kernel | sort -rV | head -n 1
2024-04-19T13:32:04.348 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod 0666 /tmp/tmp.LZlGHoHphz
2024-04-19T13:32:04.382 INFO:teuthology.orchestra.run.smithi145.stdout:kernel-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:04.382 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:04.383 DEBUG:teuthology.orchestra.run.smithi145:> sudo rpm -qi grub2-tools
2024-04-19T13:32:04.416 INFO:teuthology.orchestra.run.smithi145.stdout:Name        : grub2-tools
2024-04-19T13:32:04.417 INFO:teuthology.orchestra.run.smithi145.stdout:Epoch       : 1
2024-04-19T13:32:04.417 INFO:teuthology.orchestra.run.smithi145.stdout:Version     : 2.06
2024-04-19T13:32:04.417 INFO:teuthology.orchestra.run.smithi145.stdout:Release     : 68.el9
2024-04-19T13:32:04.417 INFO:teuthology.orchestra.run.smithi145.stdout:Architecture: x86_64
2024-04-19T13:32:04.417 INFO:teuthology.orchestra.run.smithi145.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC
2024-04-19T13:32:04.417 INFO:teuthology.orchestra.run.smithi145.stdout:Group       : Unspecified
2024-04-19T13:32:04.417 INFO:teuthology.orchestra.run.smithi145.stdout:Size        : 8264254
2024-04-19T13:32:04.417 INFO:teuthology.orchestra.run.smithi145.stdout:License     : GPLv3+
2024-04-19T13:32:04.417 INFO:teuthology.orchestra.run.smithi145.stdout:Signature   : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d
2024-04-19T13:32:04.417 INFO:teuthology.orchestra.run.smithi145.stdout:Source RPM  : grub2-2.06-68.el9.src.rpm
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:Build Date  : Wed 02 Aug 2023 03:15:48 PM UTC
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:Build Host  : x86-05.stream.rdu2.redhat.com
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:Packager    : builder@centos.org
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:Vendor      : CentOS
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:URL         : http://www.gnu.org/software/grub/
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:Summary     : Support tools for GRUB.
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:Description :
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:customizable bootloader with modular architecture.  It supports a rich
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:variety of kernel formats, file systems, computer architectures and
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:hardware devices.
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:
2024-04-19T13:32:04.418 INFO:teuthology.orchestra.run.smithi145.stdout:This subpackage provides tools for support of all platforms.
2024-04-19T13:32:04.419 INFO:teuthology.task.kernel:Updating Grub Version: grub2
2024-04-19T13:32:04.419 INFO:teuthology.task.kernel:Updating grub on smithi145 to boot 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:04.419 DEBUG:teuthology.orchestra.run.smithi145:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg
2024-04-19T13:32:04.486 DEBUG:teuthology.orchestra.remote:smithi167:/tmp/tmp.LZlGHoHphz is 6KB
2024-04-19T13:32:04.535 DEBUG:teuthology.orchestra.run.smithi167:> rm -fr /tmp/tmp.LZlGHoHphz
2024-04-19T13:32:04.549 DEBUG:teuthology.orchestra.run.smithi167:> sudo /bin/ls /boot/loader/entries || true
2024-04-19T13:32:04.624 INFO:teuthology.orchestra.run.smithi167.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf
2024-04-19T13:32:04.624 INFO:teuthology.orchestra.run.smithi167.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf
2024-04-19T13:32:04.624 INFO:teuthology.orchestra.run.smithi167.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf
2024-04-19T13:32:04.624 INFO:teuthology.orchestra.run.smithi167.stdout:15949e3c88704300882482eddfd7baae-6.9.0-0.rc2.1.el10.x86_64.conf
2024-04-19T13:32:04.625 DEBUG:teuthology.orchestra.run.smithi167:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:04.788 DEBUG:teuthology.orchestra.run.smithi167:> sudo shutdown -r now
2024-04-19T13:32:05.069 INFO:teuthology.orchestra.run.smithi145.stderr:Generating grub configuration file ...
2024-04-19T13:32:06.501 INFO:teuthology.orchestra.run.smithi145.stderr:Adding boot menu entry for UEFI Firmware Settings ...
2024-04-19T13:32:06.530 INFO:teuthology.orchestra.run.smithi145.stderr:done
2024-04-19T13:32:06.532 DEBUG:teuthology.orchestra.run.smithi145:> mktemp
2024-04-19T13:32:06.588 INFO:teuthology.orchestra.run.smithi145.stdout:/tmp/tmp.vibSo0S80J
2024-04-19T13:32:06.589 DEBUG:teuthology.orchestra.run.smithi145:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.vibSo0S80J
2024-04-19T13:32:06.666 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod 0666 /tmp/tmp.vibSo0S80J
2024-04-19T13:32:06.805 DEBUG:teuthology.orchestra.remote:smithi145:/tmp/tmp.vibSo0S80J is 6KB
2024-04-19T13:32:06.854 DEBUG:teuthology.orchestra.run.smithi145:> rm -fr /tmp/tmp.vibSo0S80J
2024-04-19T13:32:06.869 DEBUG:teuthology.orchestra.run.smithi145:> sudo /bin/ls /boot/loader/entries || true
2024-04-19T13:32:06.934 INFO:teuthology.orchestra.run.smithi145.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf
2024-04-19T13:32:06.934 INFO:teuthology.orchestra.run.smithi145.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf
2024-04-19T13:32:06.934 INFO:teuthology.orchestra.run.smithi145.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf
2024-04-19T13:32:06.934 INFO:teuthology.orchestra.run.smithi145.stdout:15949e3c88704300882482eddfd7baae-6.9.0-0.rc2.1.el10.x86_64.conf
2024-04-19T13:32:06.935 DEBUG:teuthology.orchestra.run.smithi145:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:32:07.096 DEBUG:teuthology.orchestra.run.smithi145:> sudo shutdown -r now
2024-04-19T13:32:34.818 INFO:teuthology.task.kernel:Checking client host.b for new kernel version...
2024-04-19T13:32:34.819 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi167.front.sepia.ceph.com'
2024-04-19T13:32:34.820 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T13:32:37.101 INFO:teuthology.task.kernel:Checking client host.a for new kernel version...
2024-04-19T13:32:37.101 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com'
2024-04-19T13:32:37.102 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T13:32:53.155 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.167
2024-04-19T13:32:55.462 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.145
2024-04-19T13:33:02.162 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi167.front.sepia.ceph.com'
2024-04-19T13:33:02.163 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T13:33:04.465 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com'
2024-04-19T13:33:04.474 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T13:33:05.218 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.167
2024-04-19T13:33:07.550 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.145
2024-04-19T13:33:17.226 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi167.front.sepia.ceph.com'
2024-04-19T13:33:17.227 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T13:33:19.554 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com'
2024-04-19T13:33:19.554 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T13:33:37.955 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.145
2024-04-19T13:33:52.970 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com'
2024-04-19T13:33:52.972 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T13:33:53.395 DEBUG:teuthology.orchestra.run.smithi145:> true
2024-04-19T13:33:53.831 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi145.front.sepia.ceph.com'
2024-04-19T13:33:53.831 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "6.9.0-0.rc2.1.el10.x86_64"...
2024-04-19T13:33:53.832 DEBUG:teuthology.orchestra.run.smithi145:> uname -r
2024-04-19T13:33:53.847 INFO:teuthology.orchestra.run.smithi145.stdout:6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:33:53.847 DEBUG:teuthology.task.kernel:current kernel version is 6.9.0-0.rc2.1.el10.x86_64 vs 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:33:53.848 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install
2024-04-19T13:33:53.848 DEBUG:teuthology.task.kernel:Distro of this test job: centos
2024-04-19T13:33:54.848 INFO:teuthology.task.kernel:Enabling kdb on host.a...
2024-04-19T13:33:54.849 DEBUG:teuthology.orchestra.run.smithi145:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2024-04-19T13:33:54.916 INFO:teuthology.orchestra.run.smithi145.stdout:ttyS1
2024-04-19T13:33:54.964 DEBUG:teuthology.parallel:result is None
2024-04-19T13:34:17.228 DEBUG:teuthology.orchestra.remote:timed out
2024-04-19T13:34:32.229 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi167.front.sepia.ceph.com'
2024-04-19T13:34:32.231 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T13:34:32.564 DEBUG:teuthology.orchestra.run.smithi167:> true
2024-04-19T13:34:32.958 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi167.front.sepia.ceph.com'
2024-04-19T13:34:32.958 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "6.9.0-0.rc2.1.el10.x86_64"...
2024-04-19T13:34:32.958 DEBUG:teuthology.orchestra.run.smithi167:> uname -r
2024-04-19T13:34:32.974 INFO:teuthology.orchestra.run.smithi167.stdout:6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:34:32.974 DEBUG:teuthology.task.kernel:current kernel version is 6.9.0-0.rc2.1.el10.x86_64 vs 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T13:34:32.975 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install
2024-04-19T13:34:32.975 DEBUG:teuthology.task.kernel:Distro of this test job: centos
2024-04-19T13:34:33.976 INFO:teuthology.task.kernel:Enabling kdb on host.b...
2024-04-19T13:34:33.976 DEBUG:teuthology.orchestra.run.smithi167:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2024-04-19T13:34:34.069 INFO:teuthology.orchestra.run.smithi167.stdout:ttyS1
2024-04-19T13:34:34.107 DEBUG:teuthology.parallel:result is None
2024-04-19T13:34:34.108 INFO:teuthology.run_tasks:Running task internal.base...
2024-04-19T13:34:34.115 INFO:teuthology.task.internal:Creating test directory...
2024-04-19T13:34:34.115 DEBUG:teuthology.orchestra.run.smithi145:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2024-04-19T13:34:34.119 DEBUG:teuthology.orchestra.run.smithi167:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2024-04-19T13:34:34.138 INFO:teuthology.run_tasks:Running task internal.archive_upload...
2024-04-19T13:34:34.152 INFO:teuthology.run_tasks:Running task internal.archive...
2024-04-19T13:34:34.202 INFO:teuthology.task.internal:Creating archive directory...
2024-04-19T13:34:34.202 DEBUG:teuthology.orchestra.run.smithi145:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2024-04-19T13:34:34.205 DEBUG:teuthology.orchestra.run.smithi167:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2024-04-19T13:34:34.242 INFO:teuthology.run_tasks:Running task internal.coredump...
2024-04-19T13:34:34.259 INFO:teuthology.task.internal:Enabling coredump saving...
2024-04-19T13:34:34.259 DEBUG:teuthology.orchestra.run.smithi145:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2024-04-19T13:34:34.282 DEBUG:teuthology.orchestra.run.smithi167:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2024-04-19T13:34:34.318 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-19T13:34:34.320 INFO:teuthology.orchestra.run.smithi167.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-19T13:34:34.333 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-19T13:34:34.335 INFO:teuthology.orchestra.run.smithi167.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-19T13:34:34.337 INFO:teuthology.run_tasks:Running task internal.sudo...
2024-04-19T13:34:34.346 INFO:teuthology.task.internal:Configuring sudo...
2024-04-19T13:34:34.347 DEBUG:teuthology.orchestra.run.smithi145:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2024-04-19T13:34:34.378 DEBUG:teuthology.orchestra.run.smithi167:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2024-04-19T13:34:34.415 INFO:teuthology.run_tasks:Running task internal.syslog...
2024-04-19T13:34:34.424 INFO:teuthology.task.internal.syslog:Starting syslog monitoring...
2024-04-19T13:34:34.424 DEBUG:teuthology.orchestra.run.smithi145:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2024-04-19T13:34:34.452 DEBUG:teuthology.orchestra.run.smithi167:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2024-04-19T13:34:34.473 DEBUG:teuthology.orchestra.run.smithi145:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-19T13:34:34.551 DEBUG:teuthology.orchestra.run.smithi145:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-19T13:34:34.617 DEBUG:teuthology.orchestra.run.smithi145:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-19T13:34:34.660 DEBUG:teuthology.orchestra.run.smithi145:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-19T13:34:34.707 DEBUG:teuthology.orchestra.run.smithi145:> set -ex
2024-04-19T13:34:34.707 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2024-04-19T13:34:34.776 DEBUG:teuthology.orchestra.run.smithi167:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-19T13:34:34.827 DEBUG:teuthology.orchestra.run.smithi167:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-19T13:34:34.905 DEBUG:teuthology.orchestra.run.smithi167:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-19T13:34:34.949 DEBUG:teuthology.orchestra.run.smithi167:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-19T13:34:34.998 DEBUG:teuthology.orchestra.run.smithi167:> set -ex
2024-04-19T13:34:34.998 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2024-04-19T13:34:35.071 DEBUG:teuthology.orchestra.run.smithi145:> sudo service rsyslog restart
2024-04-19T13:34:35.073 DEBUG:teuthology.orchestra.run.smithi167:> sudo service rsyslog restart
2024-04-19T13:34:35.122 INFO:teuthology.orchestra.run.smithi145.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-19T13:34:35.166 INFO:teuthology.orchestra.run.smithi167.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-19T13:34:35.606 INFO:teuthology.run_tasks:Running task internal.timer...
2024-04-19T13:34:35.615 INFO:teuthology.task.internal:Starting timer...
2024-04-19T13:34:35.615 INFO:teuthology.run_tasks:Running task pcp...
2024-04-19T13:34:35.675 INFO:teuthology.run_tasks:Running task selinux...
2024-04-19T13:34:35.720 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}
2024-04-19T13:34:35.720 DEBUG:teuthology.orchestra.run.smithi145:> sudo service auditd rotate
2024-04-19T13:34:35.801 INFO:teuthology.orchestra.run.smithi145.stdout:Rotating logs:
2024-04-19T13:34:35.803 DEBUG:teuthology.orchestra.run.smithi167:> sudo service auditd rotate
2024-04-19T13:34:35.877 INFO:teuthology.orchestra.run.smithi167.stdout:Rotating logs:
2024-04-19T13:34:35.879 DEBUG:teuthology.task.selinux:Getting current SELinux state
2024-04-19T13:34:35.879 DEBUG:teuthology.orchestra.run.smithi145:> /usr/sbin/getenforce
2024-04-19T13:34:35.903 INFO:teuthology.orchestra.run.smithi145.stdout:Permissive
2024-04-19T13:34:35.904 DEBUG:teuthology.orchestra.run.smithi167:> /usr/sbin/getenforce
2024-04-19T13:34:35.947 INFO:teuthology.orchestra.run.smithi167.stdout:Permissive
2024-04-19T13:34:35.947 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi145.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi167.front.sepia.ceph.com': 'permissive'}
2024-04-19T13:34:35.948 DEBUG:teuthology.orchestra.run.smithi145:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0
2024-04-19T13:34:35.977 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T13:34:35.978 DEBUG:teuthology.orchestra.run.smithi167:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0
2024-04-19T13:34:36.018 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T13:34:36.018 INFO:teuthology.task.selinux:Putting SELinux into permissive mode
2024-04-19T13:34:36.018 DEBUG:teuthology.orchestra.run.smithi145:> sudo /usr/sbin/setenforce permissive
2024-04-19T13:34:36.055 DEBUG:teuthology.orchestra.run.smithi167:> sudo /usr/sbin/setenforce permissive
2024-04-19T13:34:36.107 INFO:teuthology.run_tasks:Running task ansible.cephlab...
2024-04-19T13:34:36.116 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git
2024-04-19T13:34:36.122 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin
2024-04-19T13:34:36.264 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main
2024-04-19T13:34:36.288 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}]
2024-04-19T13:34:36.289 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi145.front.sepia.ceph.com,smithi167.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml
2024-04-19T13:38:30.761 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/adking-2024-04-19_13:19:23-orch:cephadm-wip-adk-testing-2024-04-17-1713-distro-default-smithi/7663451/ansible_failures.yaml
2024-04-19T13:38:30.764 ERROR:teuthology.run_tasks:Saw exception from tasks.
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 109, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 508, in begin
    super(CephLab, self).begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 342, in begin
    self.execute_playbook()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 374, in execute_playbook
    self._handle_failure(command, status)
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 402, in _handle_failure
    raise AnsibleFailedError(failures)
teuthology.exceptions.AnsibleFailedError: smithi167.front.sepia.ceph.com:  _ansible_no_log: null  changed: false  failures: []  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      cacheonly: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - dnf-utils      - sysstat      - libedit      - boost-thread      - xfsprogs      - gdisk      - parted      - libgcrypt      - fuse-libs      - openssl      - libuuid      - podman      - attr      - lsof      - gettext      - bc      - xfsdump      - blktrace      - usbredir      - valgrind      - nfs-utils      - ncurses-devel      - gcc      - git      - genisoimage      - qemu-img      - qemu-kvm-core      - qemu-kvm-block-rbd      - libacl-devel      - autoconf      - gdb      - iozone      nobest: false      releasever: null      security: false      skip_broken: false      sslverify: true      state: present      update_cache: false      update_only: false      validate_certs: true  msg: "Depsolve Error occurred: \n Problem: problem with installed package annobin-12.12-1.el9.x86_64\n\    \  - package annobin-12.12-1.el9.x86_64 from @System requires (gcc >= 11 with\    \ gcc < 12), but none of the providers can be installed\n  - package annobin-12.09-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.12-1.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - package\    \ annobin-12.28-1.el9.x86_64 from appstream requires (gcc >= 11 with gcc < 12),\    \ but none of the providers can be installed\n  - package annobin-12.31-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.31-2.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - cannot\    \ install both gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream and gcc-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both gcc-11.3.1-4.4.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.1.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-2.3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - package gcc-c++-14.0.1-0.15.el10.x86_64\    \ from CentOS-AppStream requires gcc = 14.0.1-0.15.el10, but none of the providers\    \ can be installed\n  - problem with installed package gcc-c++-11.4.1-2.1.el9.x86_64\n\    \  - package gcc-c++-11.4.1-2.1.el9.x86_64 from appstream requires libstdc++ =\    \ 11.4.1-2.1.el9, but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.1.el9.x86_64\    \ from @System requires libstdc++ = 11.4.1-2.1.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.3.1-4.4.el9.x86_64 from appstream requires\    \ libstdc++ = 11.3.1-4.4.el9, but none of the providers can be installed\n  -\    \ package gcc-c++-11.4.1-2.3.el9.x86_64 from appstream requires libstdc++ = 11.4.1-2.3.el9,\    \ but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.el9.x86_64\    \ from appstream requires libstdc++ = 11.4.1-2.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.4.1-3.el9.x86_64 from appstream requires\    \ libstdc++ = 11.4.1-3.el9, but none of the providers can be installed\n  - libstdc++-11.4.1-2.1.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.3.1-4.4.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - cannot install\    \ both libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS and libstdc++-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both libstdc++-11.3.1-4.4.el9.x86_64 from baseos\    \ and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot install\    \ both libstdc++-11.4.1-2.1.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-2.3.el9.x86_64\    \ from baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot\    \ install both libstdc++-11.4.1-2.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-3.el9.x86_64 from\    \ baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - package\    \ python3-hawkey-0.73.0-1.el10.x86_64 from CentOS-BaseOS requires libstdc++.so.6(GLIBCXX_3.4.30)(64bit),\    \ but none of the providers can be installed\n  - package python3-dnf-4.19.0-2.el10.noarch\    \ from CentOS-BaseOS requires python3-hawkey >= 0.73.0, but none of the providers\    \ can be installed\n  - package yum-utils-4.5.0-1.el10.noarch from CentOS-BaseOS\    \ requires python3-dnf >= 4.19.0, but none of the providers can be installed\n\    \  - cannot install the best candidate for the job"  rc: 1  results: []smithi145.front.sepia.ceph.com:  _ansible_no_log: null  changed: false  failures: []  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      cacheonly: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - dnf-utils      - sysstat      - libedit      - boost-thread      - xfsprogs      - gdisk      - parted      - libgcrypt      - fuse-libs      - openssl      - libuuid      - podman      - attr      - lsof      - gettext      - bc      - xfsdump      - blktrace      - usbredir      - valgrind      - nfs-utils      - ncurses-devel      - gcc      - git      - genisoimage      - qemu-img      - qemu-kvm-core      - qemu-kvm-block-rbd      - libacl-devel      - autoconf      - gdb      - iozone      nobest: false      releasever: null      security: false      skip_broken: false      sslverify: true      state: present      update_cache: false      update_only: false      validate_certs: true  msg: "Depsolve Error occurred: \n Problem: problem with installed package annobin-12.12-1.el9.x86_64\n\    \  - package annobin-12.12-1.el9.x86_64 from @System requires (gcc >= 11 with\    \ gcc < 12), but none of the providers can be installed\n  - package annobin-12.09-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.12-1.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - package\    \ annobin-12.28-1.el9.x86_64 from appstream requires (gcc >= 11 with gcc < 12),\    \ but none of the providers can be installed\n  - package annobin-12.31-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.31-2.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - cannot\    \ install both gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream and gcc-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both gcc-11.3.1-4.4.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.1.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-2.3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - package gcc-c++-14.0.1-0.15.el10.x86_64\    \ from CentOS-AppStream requires gcc = 14.0.1-0.15.el10, but none of the providers\    \ can be installed\n  - problem with installed package gcc-c++-11.4.1-2.1.el9.x86_64\n\    \  - package gcc-c++-11.4.1-2.1.el9.x86_64 from appstream requires libstdc++ =\    \ 11.4.1-2.1.el9, but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.1.el9.x86_64\    \ from @System requires libstdc++ = 11.4.1-2.1.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.3.1-4.4.el9.x86_64 from appstream requires\    \ libstdc++ = 11.3.1-4.4.el9, but none of the providers can be installed\n  -\    \ package gcc-c++-11.4.1-2.3.el9.x86_64 from appstream requires libstdc++ = 11.4.1-2.3.el9,\    \ but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.el9.x86_64\    \ from appstream requires libstdc++ = 11.4.1-2.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.4.1-3.el9.x86_64 from appstream requires\    \ libstdc++ = 11.4.1-3.el9, but none of the providers can be installed\n  - libstdc++-11.4.1-2.1.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.3.1-4.4.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - cannot install\    \ both libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS and libstdc++-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both libstdc++-11.3.1-4.4.el9.x86_64 from baseos\    \ and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot install\    \ both libstdc++-11.4.1-2.1.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-2.3.el9.x86_64\    \ from baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot\    \ install both libstdc++-11.4.1-2.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-3.el9.x86_64 from\    \ baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - package\    \ python3-hawkey-0.73.0-1.el10.x86_64 from CentOS-BaseOS requires libstdc++.so.6(GLIBCXX_3.4.30)(64bit),\    \ but none of the providers can be installed\n  - package python3-dnf-4.19.0-2.el10.noarch\    \ from CentOS-BaseOS requires python3-hawkey >= 0.73.0, but none of the providers\    \ can be installed\n  - package yum-utils-4.5.0-1.el10.noarch from CentOS-BaseOS\    \ requires python3-dnf >= 4.19.0, but none of the providers can be installed\n\    \  - cannot install the best candidate for the job"  rc: 1  results: []
2024-04-19T13:38:30.991 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=c7b84b02b466459195baa492f0533fdd
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 109, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 508, in begin
    super(CephLab, self).begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 342, in begin
    self.execute_playbook()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 374, in execute_playbook
    self._handle_failure(command, status)
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 402, in _handle_failure
    raise AnsibleFailedError(failures)
teuthology.exceptions.AnsibleFailedError: smithi167.front.sepia.ceph.com:  _ansible_no_log: null  changed: false  failures: []  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      cacheonly: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - dnf-utils      - sysstat      - libedit      - boost-thread      - xfsprogs      - gdisk      - parted      - libgcrypt      - fuse-libs      - openssl      - libuuid      - podman      - attr      - lsof      - gettext      - bc      - xfsdump      - blktrace      - usbredir      - valgrind      - nfs-utils      - ncurses-devel      - gcc      - git      - genisoimage      - qemu-img      - qemu-kvm-core      - qemu-kvm-block-rbd      - libacl-devel      - autoconf      - gdb      - iozone      nobest: false      releasever: null      security: false      skip_broken: false      sslverify: true      state: present      update_cache: false      update_only: false      validate_certs: true  msg: "Depsolve Error occurred: \n Problem: problem with installed package annobin-12.12-1.el9.x86_64\n\    \  - package annobin-12.12-1.el9.x86_64 from @System requires (gcc >= 11 with\    \ gcc < 12), but none of the providers can be installed\n  - package annobin-12.09-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.12-1.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - package\    \ annobin-12.28-1.el9.x86_64 from appstream requires (gcc >= 11 with gcc < 12),\    \ but none of the providers can be installed\n  - package annobin-12.31-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.31-2.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - cannot\    \ install both gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream and gcc-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both gcc-11.3.1-4.4.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.1.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-2.3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - package gcc-c++-14.0.1-0.15.el10.x86_64\    \ from CentOS-AppStream requires gcc = 14.0.1-0.15.el10, but none of the providers\    \ can be installed\n  - problem with installed package gcc-c++-11.4.1-2.1.el9.x86_64\n\    \  - package gcc-c++-11.4.1-2.1.el9.x86_64 from appstream requires libstdc++ =\    \ 11.4.1-2.1.el9, but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.1.el9.x86_64\    \ from @System requires libstdc++ = 11.4.1-2.1.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.3.1-4.4.el9.x86_64 from appstream requires\    \ libstdc++ = 11.3.1-4.4.el9, but none of the providers can be installed\n  -\    \ package gcc-c++-11.4.1-2.3.el9.x86_64 from appstream requires libstdc++ = 11.4.1-2.3.el9,\    \ but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.el9.x86_64\    \ from appstream requires libstdc++ = 11.4.1-2.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.4.1-3.el9.x86_64 from appstream requires\    \ libstdc++ = 11.4.1-3.el9, but none of the providers can be installed\n  - libstdc++-11.4.1-2.1.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.3.1-4.4.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - cannot install\    \ both libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS and libstdc++-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both libstdc++-11.3.1-4.4.el9.x86_64 from baseos\    \ and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot install\    \ both libstdc++-11.4.1-2.1.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-2.3.el9.x86_64\    \ from baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot\    \ install both libstdc++-11.4.1-2.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-3.el9.x86_64 from\    \ baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - package\    \ python3-hawkey-0.73.0-1.el10.x86_64 from CentOS-BaseOS requires libstdc++.so.6(GLIBCXX_3.4.30)(64bit),\    \ but none of the providers can be installed\n  - package python3-dnf-4.19.0-2.el10.noarch\    \ from CentOS-BaseOS requires python3-hawkey >= 0.73.0, but none of the providers\    \ can be installed\n  - package yum-utils-4.5.0-1.el10.noarch from CentOS-BaseOS\    \ requires python3-dnf >= 4.19.0, but none of the providers can be installed\n\    \  - cannot install the best candidate for the job"  rc: 1  results: []smithi145.front.sepia.ceph.com:  _ansible_no_log: null  changed: false  failures: []  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      cacheonly: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - dnf-utils      - sysstat      - libedit      - boost-thread      - xfsprogs      - gdisk      - parted      - libgcrypt      - fuse-libs      - openssl      - libuuid      - podman      - attr      - lsof      - gettext      - bc      - xfsdump      - blktrace      - usbredir      - valgrind      - nfs-utils      - ncurses-devel      - gcc      - git      - genisoimage      - qemu-img      - qemu-kvm-core      - qemu-kvm-block-rbd      - libacl-devel      - autoconf      - gdb      - iozone      nobest: false      releasever: null      security: false      skip_broken: false      sslverify: true      state: present      update_cache: false      update_only: false      validate_certs: true  msg: "Depsolve Error occurred: \n Problem: problem with installed package annobin-12.12-1.el9.x86_64\n\    \  - package annobin-12.12-1.el9.x86_64 from @System requires (gcc >= 11 with\    \ gcc < 12), but none of the providers can be installed\n  - package annobin-12.09-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.12-1.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - package\    \ annobin-12.28-1.el9.x86_64 from appstream requires (gcc >= 11 with gcc < 12),\    \ but none of the providers can be installed\n  - package annobin-12.31-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.31-2.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - cannot\    \ install both gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream and gcc-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both gcc-11.3.1-4.4.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.1.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-2.3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - package gcc-c++-14.0.1-0.15.el10.x86_64\    \ from CentOS-AppStream requires gcc = 14.0.1-0.15.el10, but none of the providers\    \ can be installed\n  - problem with installed package gcc-c++-11.4.1-2.1.el9.x86_64\n\    \  - package gcc-c++-11.4.1-2.1.el9.x86_64 from appstream requires libstdc++ =\    \ 11.4.1-2.1.el9, but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.1.el9.x86_64\    \ from @System requires libstdc++ = 11.4.1-2.1.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.3.1-4.4.el9.x86_64 from appstream requires\    \ libstdc++ = 11.3.1-4.4.el9, but none of the providers can be installed\n  -\    \ package gcc-c++-11.4.1-2.3.el9.x86_64 from appstream requires libstdc++ = 11.4.1-2.3.el9,\    \ but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.el9.x86_64\    \ from appstream requires libstdc++ = 11.4.1-2.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.4.1-3.el9.x86_64 from appstream requires\    \ libstdc++ = 11.4.1-3.el9, but none of the providers can be installed\n  - libstdc++-11.4.1-2.1.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.3.1-4.4.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - cannot install\    \ both libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS and libstdc++-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both libstdc++-11.3.1-4.4.el9.x86_64 from baseos\    \ and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot install\    \ both libstdc++-11.4.1-2.1.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-2.3.el9.x86_64\    \ from baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot\    \ install both libstdc++-11.4.1-2.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-3.el9.x86_64 from\    \ baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - package\    \ python3-hawkey-0.73.0-1.el10.x86_64 from CentOS-BaseOS requires libstdc++.so.6(GLIBCXX_3.4.30)(64bit),\    \ but none of the providers can be installed\n  - package python3-dnf-4.19.0-2.el10.noarch\    \ from CentOS-BaseOS requires python3-hawkey >= 0.73.0, but none of the providers\    \ can be installed\n  - package yum-utils-4.5.0-1.el10.noarch from CentOS-BaseOS\    \ requires python3-dnf >= 4.19.0, but none of the providers can be installed\n\    \  - cannot install the best candidate for the job"  rc: 1  results: []
2024-04-19T13:38:30.993 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab
2024-04-19T13:38:31.002 INFO:teuthology.task.ansible:Skipping ansible cleanup...
2024-04-19T13:38:31.003 DEBUG:teuthology.run_tasks:Unwinding manager selinux
2024-04-19T13:38:31.042 DEBUG:teuthology.orchestra.run.smithi145:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log
2024-04-19T13:38:31.113 DEBUG:teuthology.orchestra.run.smithi167:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log
2024-04-19T13:38:31.179 DEBUG:teuthology.orchestra.run.smithi145:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0
2024-04-19T13:38:31.211 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T13:38:31.212 DEBUG:teuthology.orchestra.run.smithi167:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0
2024-04-19T13:38:31.244 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T13:38:31.245 DEBUG:teuthology.run_tasks:Unwinding manager pcp
2024-04-19T13:38:31.257 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer
2024-04-19T13:38:31.284 INFO:teuthology.task.internal:Duration was 235.669172 seconds
2024-04-19T13:38:31.284 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog
2024-04-19T13:38:31.333 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring...
2024-04-19T13:38:31.333 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart
2024-04-19T13:38:31.336 DEBUG:teuthology.orchestra.run.smithi167:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart
2024-04-19T13:38:31.382 INFO:teuthology.orchestra.run.smithi145.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-19T13:38:31.384 INFO:teuthology.orchestra.run.smithi167.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-19T13:38:31.868 INFO:teuthology.task.internal.syslog:Checking logs for errors...
2024-04-19T13:38:31.868 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi145.front.sepia.ceph.com
2024-04-19T13:38:31.868 DEBUG:teuthology.orchestra.run.smithi145:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1
2024-04-19T13:38:31.911 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi167.front.sepia.ceph.com
2024-04-19T13:38:31.911 DEBUG:teuthology.orchestra.run.smithi167:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1
2024-04-19T13:38:31.946 INFO:teuthology.task.internal.syslog:Compressing syslogs...
2024-04-19T13:38:31.946 DEBUG:teuthology.orchestra.run.smithi145:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
2024-04-19T13:38:31.954 DEBUG:teuthology.orchestra.run.smithi167:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
2024-04-19T13:38:32.017 INFO:teuthology.task.internal.syslog:Gathering journactl -b0...
2024-04-19T13:38:32.018 DEBUG:teuthology.orchestra.run.smithi145:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz
2024-04-19T13:38:32.027 DEBUG:teuthology.orchestra.run.smithi167:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz
2024-04-19T13:38:32.144 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo
2024-04-19T13:38:32.152 INFO:teuthology.task.internal:Restoring /etc/sudoers...
2024-04-19T13:38:32.153 DEBUG:teuthology.orchestra.run.smithi145:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers
2024-04-19T13:38:32.181 DEBUG:teuthology.orchestra.run.smithi167:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers
2024-04-19T13:38:32.210 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
2024-04-19T13:38:32.222 DEBUG:teuthology.orchestra.run.smithi145:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
2024-04-19T13:38:32.226 DEBUG:teuthology.orchestra.run.smithi167:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
2024-04-19T13:38:32.250 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern = core
2024-04-19T13:38:32.277 INFO:teuthology.orchestra.run.smithi167.stdout:kernel.core_pattern = core
2024-04-19T13:38:32.302 DEBUG:teuthology.orchestra.run.smithi145:> test -e /home/ubuntu/cephtest/archive/coredump
2024-04-19T13:38:32.341 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T13:38:32.342 DEBUG:teuthology.orchestra.run.smithi167:> test -e /home/ubuntu/cephtest/archive/coredump
2024-04-19T13:38:32.356 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T13:38:32.357 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive
2024-04-19T13:38:32.366 INFO:teuthology.task.internal:Transferring archived files...
2024-04-19T13:38:32.368 DEBUG:teuthology.misc:Transferring archived files from smithi145:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-19_13:19:23-orch:cephadm-wip-adk-testing-2024-04-17-1713-distro-default-smithi/7663451/remote/smithi145
2024-04-19T13:38:32.369 DEBUG:teuthology.orchestra.run.smithi145:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- .
2024-04-19T13:38:32.437 DEBUG:teuthology.misc:Transferring archived files from smithi167:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-19_13:19:23-orch:cephadm-wip-adk-testing-2024-04-17-1713-distro-default-smithi/7663451/remote/smithi167
2024-04-19T13:38:32.438 DEBUG:teuthology.orchestra.run.smithi167:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- .
2024-04-19T13:38:32.483 INFO:teuthology.task.internal:Removing archive directory...
2024-04-19T13:38:32.483 DEBUG:teuthology.orchestra.run.smithi145:> rm -rf -- /home/ubuntu/cephtest/archive
2024-04-19T13:38:32.485 DEBUG:teuthology.orchestra.run.smithi167:> rm -rf -- /home/ubuntu/cephtest/archive
2024-04-19T13:38:32.534 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload
2024-04-19T13:38:32.545 INFO:teuthology.task.internal:Not uploading archives.
2024-04-19T13:38:32.545 DEBUG:teuthology.run_tasks:Unwinding manager internal.base
2024-04-19T13:38:32.576 INFO:teuthology.task.internal:Tidying up after the test...
2024-04-19T13:38:32.577 DEBUG:teuthology.orchestra.run.smithi145:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest
2024-04-19T13:38:32.579 DEBUG:teuthology.orchestra.run.smithi167:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest
2024-04-19T13:38:32.592 INFO:teuthology.orchestra.run.smithi145.stdout:   262166      4 drwxr-xr-x   2 ubuntu   ubuntu       4096 Apr 19 13:38 /home/ubuntu/cephtest
2024-04-19T13:38:32.595 INFO:teuthology.orchestra.run.smithi167.stdout:   262166      4 drwxr-xr-x   2 ubuntu   ubuntu       4096 Apr 19 13:38 /home/ubuntu/cephtest
2024-04-19T13:38:32.596 DEBUG:teuthology.run_tasks:Unwinding manager kernel
2024-04-19T13:38:32.627 DEBUG:teuthology.run_tasks:Unwinding manager console_log
2024-04-19T13:38:32.727 INFO:teuthology.nuke:Checking targets against current locks
2024-04-19T13:38:32.755 DEBUG:teuthology.nuke:shortname: smithi145
2024-04-19T13:38:32.756 INFO:teuthology.task.internal.check_lock:Checking locks...
2024-04-19T13:38:32.781 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi145.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-19_13:19:23-orch:cephadm-wip-adk-testing-2024-04-17-1713-distro-default-smithi/7663451', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-19 13:20:48.544443', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMFEFBaZCyv0FZIYfZt1oWSZLs8KGDjkh5OlrfdBKZN6JkPcE1aXaBstW48vAZOvwvcU6ra46GAlaM9dfXx/B+M='}
2024-04-19T13:38:32.786 DEBUG:teuthology.nuke:shortname: smithi167
2024-04-19T13:38:32.786 INFO:teuthology.task.internal.check_lock:Checking locks...
2024-04-19T13:38:32.808 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi167.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-19_13:19:23-orch:cephadm-wip-adk-testing-2024-04-17-1713-distro-default-smithi/7663451', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-19 13:20:48.545681', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHopHKQ3L0HXp/0IHScTj04oJEJcsVCW8FOFOuoUVmGC0a9R31+15ldw0ZDRmSV0YjhRkug6BwogL0mxOQMPXgQ='}
2024-04-19T13:38:32.838 INFO:teuthology.orchestra.console.smithi145:Power off
2024-04-19T13:38:32.838 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2024-04-19T13:38:32.865 DEBUG:teuthology.orchestra.console.smithi145:power off output: Chassis Power Control: Down/Off
2024-04-19T13:38:32.865 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T13:38:32.882 INFO:teuthology.orchestra.console.smithi167:Power off
2024-04-19T13:38:32.882 DEBUG:teuthology.orchestra.console.smithi167:pexpect command: ipmitool -H smithi167.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2024-04-19T13:38:32.893 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is on
2024-04-19T13:38:32.907 DEBUG:teuthology.orchestra.console.smithi167:power off output: Chassis Power Control: Down/Off
2024-04-19T13:38:32.907 DEBUG:teuthology.orchestra.console.smithi167:pexpect command: ipmitool -H smithi167.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T13:38:32.937 DEBUG:teuthology.orchestra.console.smithi167:check power output: Chassis Power is on
2024-04-19T13:38:36.894 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T13:38:36.938 DEBUG:teuthology.orchestra.console.smithi167:pexpect command: ipmitool -H smithi167.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T13:38:37.006 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is on
2024-04-19T13:38:37.048 DEBUG:teuthology.orchestra.console.smithi167:check power output: Chassis Power is on
2024-04-19T13:38:41.012 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T13:38:41.049 DEBUG:teuthology.orchestra.console.smithi167:pexpect command: ipmitool -H smithi167.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T13:38:41.125 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is off
2024-04-19T13:38:41.160 DEBUG:teuthology.orchestra.console.smithi167:check power output: Chassis Power is on
2024-04-19T13:38:41.227 INFO:teuthology.orchestra.console.smithi145:Power off completed
2024-04-19T13:38:45.164 DEBUG:teuthology.orchestra.console.smithi167:pexpect command: ipmitool -H smithi167.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T13:38:45.273 DEBUG:teuthology.orchestra.console.smithi167:check power output: Chassis Power is off
2024-04-19T13:38:45.374 INFO:teuthology.orchestra.console.smithi167:Power off completed
2024-04-19T13:38:45.506 INFO:teuthology.run:Summary data:
description: orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc 0-nvme-loop
  1-start 2-services/nfs-ingress2 3-final}
duration: 235.66917157173157
failure_reason: 'smithi167.front.sepia.ceph.com:  _ansible_no_log: null  changed:
  false  failures: []  invocation:    module_args:      allow_downgrade: false      allowerasing:
  false      autoremove: false      bugfix: false      cacheonly: false      conf_file:
  null      disable_excludes: null      disable_gpg_check: false      disable_plugin:
  []      disablerepo: []      download_dir: null      download_only: false      enable_plugin:
  []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps:
  true      installroot: /      list: null      lock_timeout: 30      name:      -
  dnf-utils      - sysstat      - libedit      - boost-thread      - xfsprogs      -
  gdisk      - parted      - libgcrypt      - fuse-libs      - openssl      - libuuid      -
  podman      - attr      - lsof      - gettext      - bc      - xfsdump      - blktrace      -
  usbredir      - valgrind      - nfs-utils      - ncurses-devel      - gcc      -
  git      - genisoimage      - qemu-img      - qemu-kvm-core      - qemu-kvm-block-rbd      -
  libacl-devel      - autoconf      - gdb      - iozone      nobest: false      releasever:
  null      security: false      skip_broken: false      sslverify: true      state:
  present      update_cache: false      update_only: false      validate_certs: true  msg:
  "Depsolve Error occurred: \n Problem: problem with installed package annobin-12.12-1.el9.x86_64\n\    \  -
  package annobin-12.12-1.el9.x86_64 from @System requires (gcc >= 11 with\    \ gcc
  < 12), but none of the providers can be installed\n  - package annobin-12.09-1.el9.x86_64\    \
  from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \
  can be installed\n  - package annobin-12.12-1.el9.x86_64 from appstream requires\    \
  (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - package\    \
  annobin-12.28-1.el9.x86_64 from appstream requires (gcc >= 11 with gcc < 12),\    \
  but none of the providers can be installed\n  - package annobin-12.31-1.el9.x86_64\    \
  from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \
  can be installed\n  - package annobin-12.31-2.el9.x86_64 from appstream requires\    \
  (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - cannot\    \
  install both gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream and gcc-11.4.1-2.1.el9.x86_64\    \
  from @System\n  - cannot install both gcc-11.3.1-4.4.el9.x86_64 from appstream\    \
  and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \
  gcc-11.4.1-2.1.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \
  CentOS-AppStream\n  - cannot install both gcc-11.4.1-2.3.el9.x86_64 from appstream\    \
  and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \
  gcc-11.4.1-2.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \
  CentOS-AppStream\n  - cannot install both gcc-11.4.1-3.el9.x86_64 from appstream\    \
  and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - package gcc-c++-14.0.1-0.15.el10.x86_64\    \
  from CentOS-AppStream requires gcc = 14.0.1-0.15.el10, but none of the providers\    \
  can be installed\n  - problem with installed package gcc-c++-11.4.1-2.1.el9.x86_64\n\    \  -
  package gcc-c++-11.4.1-2.1.el9.x86_64 from appstream requires libstdc++ =\    \
  11.4.1-2.1.el9, but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.1.el9.x86_64\    \
  from @System requires libstdc++ = 11.4.1-2.1.el9, but none of the providers\    \
  can be installed\n  - package gcc-c++-11.3.1-4.4.el9.x86_64 from appstream requires\    \
  libstdc++ = 11.3.1-4.4.el9, but none of the providers can be installed\n  -\    \
  package gcc-c++-11.4.1-2.3.el9.x86_64 from appstream requires libstdc++ = 11.4.1-2.3.el9,\    \
  but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.el9.x86_64\    \
  from appstream requires libstdc++ = 11.4.1-2.el9, but none of the providers\    \
  can be installed\n  - package gcc-c++-11.4.1-3.el9.x86_64 from appstream requires\    \
  libstdc++ = 11.4.1-3.el9, but none of the providers can be installed\n  - libstdc++-11.4.1-2.1.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.3.1-4.4.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.3.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-3.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - cannot install\    \
  both libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS and libstdc++-11.4.1-2.1.el9.x86_64\    \
  from @System\n  - cannot install both libstdc++-11.3.1-4.4.el9.x86_64 from baseos\    \
  and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot install\    \
  both libstdc++-11.4.1-2.1.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \
  from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-2.3.el9.x86_64\    \
  from baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot\    \
  install both libstdc++-11.4.1-2.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \
  from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-3.el9.x86_64 from\    \
  baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - package\    \
  python3-hawkey-0.73.0-1.el10.x86_64 from CentOS-BaseOS requires libstdc++.so.6(GLIBCXX_3.4.30)(64bit),\    \
  but none of the providers can be installed\n  - package python3-dnf-4.19.0-2.el10.noarch\    \
  from CentOS-BaseOS requires python3-hawkey >= 0.73.0, but none of the providers\    \
  can be installed\n  - package yum-utils-4.5.0-1.el10.noarch from CentOS-BaseOS\    \
  requires python3-dnf >= 4.19.0, but none of the providers can be installed\n\    \  -
  cannot install the best candidate for the job"  rc: 1  results: []smithi145.front.sepia.ceph.com:  _ansible_no_log:
  null  changed: false  failures: []  invocation:    module_args:      allow_downgrade:
  false      allowerasing: false      autoremove: false      bugfix: false      cacheonly:
  false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin:
  []      disablerepo: []      download_dir: null      download_only: false      enable_plugin:
  []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps:
  true      installroot: /      list: null      lock_timeout: 30      name:      -
  dnf-utils      - sysstat      - libedit      - boost-thread      - xfsprogs      -
  gdisk      - parted      - libgcrypt      - fuse-libs      - openssl      - libuuid      -
  podman      - attr      - lsof      - gettext      - bc      - xfsdump      - blktrace      -
  usbredir      - valgrind      - nfs-utils      - ncurses-devel      - gcc      -
  git      - genisoimage      - qemu-img      - qemu-kvm-core      - qemu-kvm-block-rbd      -
  libacl-devel      - autoconf      - gdb      - iozone      nobest: false      releasever:
  null      security: false      skip_broken: false      sslverify: true      state:
  present      update_cache: false      update_only: false      validate_certs: true  msg:
  "Depsolve Error occurred: \n Problem: problem with installed package annobin-12.12-1.el9.x86_64\n\    \  -
  package annobin-12.12-1.el9.x86_64 from @System requires (gcc >= 11 with\    \ gcc
  < 12), but none of the providers can be installed\n  - package annobin-12.09-1.el9.x86_64\    \
  from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \
  can be installed\n  - package annobin-12.12-1.el9.x86_64 from appstream requires\    \
  (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - package\    \
  annobin-12.28-1.el9.x86_64 from appstream requires (gcc >= 11 with gcc < 12),\    \
  but none of the providers can be installed\n  - package annobin-12.31-1.el9.x86_64\    \
  from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \
  can be installed\n  - package annobin-12.31-2.el9.x86_64 from appstream requires\    \
  (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - cannot\    \
  install both gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream and gcc-11.4.1-2.1.el9.x86_64\    \
  from @System\n  - cannot install both gcc-11.3.1-4.4.el9.x86_64 from appstream\    \
  and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \
  gcc-11.4.1-2.1.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \
  CentOS-AppStream\n  - cannot install both gcc-11.4.1-2.3.el9.x86_64 from appstream\    \
  and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \
  gcc-11.4.1-2.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \
  CentOS-AppStream\n  - cannot install both gcc-11.4.1-3.el9.x86_64 from appstream\    \
  and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - package gcc-c++-14.0.1-0.15.el10.x86_64\    \
  from CentOS-AppStream requires gcc = 14.0.1-0.15.el10, but none of the providers\    \
  can be installed\n  - problem with installed package gcc-c++-11.4.1-2.1.el9.x86_64\n\    \  -
  package gcc-c++-11.4.1-2.1.el9.x86_64 from appstream requires libstdc++ =\    \
  11.4.1-2.1.el9, but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.1.el9.x86_64\    \
  from @System requires libstdc++ = 11.4.1-2.1.el9, but none of the providers\    \
  can be installed\n  - package gcc-c++-11.3.1-4.4.el9.x86_64 from appstream requires\    \
  libstdc++ = 11.3.1-4.4.el9, but none of the providers can be installed\n  -\    \
  package gcc-c++-11.4.1-2.3.el9.x86_64 from appstream requires libstdc++ = 11.4.1-2.3.el9,\    \
  but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.el9.x86_64\    \
  from appstream requires libstdc++ = 11.4.1-2.el9, but none of the providers\    \
  can be installed\n  - package gcc-c++-11.4.1-3.el9.x86_64 from appstream requires\    \
  libstdc++ = 11.4.1-3.el9, but none of the providers can be installed\n  - libstdc++-11.4.1-2.1.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.3.1-4.4.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.3.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-3.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - cannot install\    \
  both libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS and libstdc++-11.4.1-2.1.el9.x86_64\    \
  from @System\n  - cannot install both libstdc++-11.3.1-4.4.el9.x86_64 from baseos\    \
  and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot install\    \
  both libstdc++-11.4.1-2.1.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \
  from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-2.3.el9.x86_64\    \
  from baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot\    \
  install both libstdc++-11.4.1-2.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \
  from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-3.el9.x86_64 from\    \
  baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - package\    \
  python3-hawkey-0.73.0-1.el10.x86_64 from CentOS-BaseOS requires libstdc++.so.6(GLIBCXX_3.4.30)(64bit),\    \
  but none of the providers can be installed\n  - package python3-dnf-4.19.0-2.el10.noarch\    \
  from CentOS-BaseOS requires python3-hawkey >= 0.73.0, but none of the providers\    \
  can be installed\n  - package yum-utils-4.5.0-1.el10.noarch from CentOS-BaseOS\    \
  requires python3-dnf >= 4.19.0, but none of the providers can be installed\n\    \  -
  cannot install the best candidate for the job"  rc: 1  results: []'
owner: scheduled_adking@teuthology
sentry_event: https://sentry.ceph.com/organizations/ceph/?query=c7b84b02b466459195baa492f0533fdd
status: dead
success: false

2024-04-19T13:38:45.506 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/
2024-04-19T13:38:45.586 INFO:teuthology.run:DEAD