2024-04-19T02:30:58.848 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d
2024-04-19T02:30:58.848 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2024-04-19_02:08:09-upgrade-squid-distro-default-smithi/7663374 --verbose --name teuthology-2024-04-19_02:08:09-upgrade-squid-distro-default-smithi --owner scheduled_teuthology@teuthology --description upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} -- /home/teuthworker/archive/teuthology-2024-04-19_02:08:09-upgrade-squid-distro-default-smithi/7663374/orig.config.yaml
2024-04-19T02:30:58.903 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/
2024-04-19T02:30:59.010 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/teuthology-2024-04-19_02:08:09-upgrade-squid-distro-default-smithi/7663374
  branch: squid
  description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start
    2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export
    rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
  email: ceph-qa@ceph.com
  first-half-sequence:
  - cephadm.shell:
      env:
      - sha1
      mon.a:
      - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60
      - ceph config set global log_to_journald false --force
      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
      - ceph orch ps
      - echo wait for minority of mons to upgrade
      - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done
      - ceph orch ps
      - ceph orch upgrade pause
      - sleep 60
      - ceph orch upgrade resume
      - echo wait for majority of mons to upgrade
      - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done'
      - ceph orch ps
      - ceph orch upgrade pause
      - sleep 60
      - ceph orch upgrade resume
      - echo wait for all mons to upgrade
      - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done'
      - ceph orch ps
      - ceph orch upgrade pause
      - sleep 60
      - ceph orch upgrade resume
      - echo wait for half of osds to upgrade
      - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ;
        done'
      - ceph orch upgrade pause
      - ceph orch ps
      - ceph orch ps
      - ceph versions
  first-half-tasks:
  - full_sequential:
    - rados:
        clients:
        - client.0
        objects: 50
        op_weights:
          delete: 50
          read: 100
          rollback: 50
          snap_create: 50
          snap_remove: 50
          write: 100
        ops: 4000
        write_append_excl: false
  - print: '**** done rados/snaps-few-objects 5-workload'
  first_in_suite: false
  job_id: '7663374'
  kernel:
    kdb: 1
    sha1: distro
  last_in_suite: false
  machine_type: smithi
  meta:
  - desc: 'randomized correctness test for rados operations on a replicated pool with
      snapshot operations
  
      '
  - desc: 'run randomized correctness test for rados operations
  
      generate write load with rados bench
  
      '
  - desc: 'run basic cls tests for rbd
  
      '
  - desc: 'run basic import/export cli tests for rbd
  
      '
  - desc: 'librbd C and C++ api tests
  
      '
  - desc: 'randomized correctness test for rados operations on a replicated pool,
  
      using only reads, writes, and deletes
  
      '
  - desc: 'randomized correctness test for rados operations on a replicated pool with
      snapshot operations
  
      '
  - desc: 'run randomized correctness test for rados operations
  
      generate write load with rados bench
  
      '
  name: teuthology-2024-04-19_02:08:09-upgrade-squid-distro-default-smithi
  no_nested_subset: false
  nuke-on-error: true
  openstack:
  - volumes:
      count: 4
      size: 10
  os_type: centos
  os_version: 9.stream
  overrides:
    admin_socket:
      branch: squid
    ceph:
      conf:
        global:
          mon election default strategy: 1
        mgr:
          debug mgr: 20
          debug ms: 1
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
        osd:
          debug ms: 1
          debug osd: 20
          osd shutdown pgref assert: true
      create_rbd_pool: true
      flavor: default
      log-ignorelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      sha1: 1a6a3e742fd81c6d70be70552499bff767652c32
    ceph-deploy:
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon: {}
    install:
      ceph:
        flavor: default
        sha1: 1a6a3e742fd81c6d70be70552499bff767652c32
    selinux:
      allowlist:
      - scontext=system_u:system_r:logrotate_t:s0
    workunit:
      branch: squid
      sha1: d0bf63dd179c05751e4be49477389aa290c73089
  owner: scheduled_teuthology@teuthology
  priority: 100
  repo: https://git.ceph.com/ceph.git
  roles:
  - - mon.a
    - mon.c
    - mgr.y
    - osd.0
    - osd.1
    - osd.2
    - osd.3
    - client.0
    - node-exporter.a
    - alertmanager.a
  - - mon.b
    - mgr.x
    - osd.4
    - osd.5
    - osd.6
    - osd.7
    - client.1
    - prometheus.a
    - grafana.a
    - node-exporter.b
  second-half-sequence:
    sequential:
    - cephadm.shell:
        env:
        - sha1
        mon.a:
        - ceph orch upgrade resume
        - sleep 60
        - echo wait for upgrade to complete
        - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch
          ps ; ceph versions ; sleep 30 ; done
        - echo upgrade complete
        - ceph orch ps
        - ceph versions
        - ceph versions | jq -e '.overall | length == 1'
        - ceph versions | jq -e '.overall | keys' | grep $sha1
  second-half-tasks:
  - full_sequential:
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
  - print: '**** done end radosbench.yaml'
  seed: 7092
  sha1: 1a6a3e742fd81c6d70be70552499bff767652c32
  sleep_before_teardown: 0
  stress-tasks:
  - thrashosds:
      aggressive_pg_num_changes: false
      chance_force_recovery: 0
      chance_pgnum_grow: 1
      chance_pgpnum_fix: 1
      chance_thrash_cluster_full: 0
      chance_thrash_pg_upmap: 0
      chance_thrash_pg_upmap_items: 0
      disable_objectstore_tool_tests: true
      timeout: 1200
  - full_sequential:
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
  - print: '**** done end radosbench.yaml'
  - workunit:
      branch: reef
      clients:
        client.0:
        - cls/test_cls_rbd.sh
  - print: '**** done cls/test_cls_rbd.sh 5-workload'
  - workunit:
      branch: reef
      clients:
        client.0:
        - rbd/import_export.sh
      env:
        RBD_CREATE_ARGS: --new-format
  - print: '**** done rbd/import_export.sh 5-workload'
  - workunit:
      branch: reef
      clients:
        client.0:
        - rbd/test_librbd.sh
      env:
        RBD_FEATURES: '61'
  - print: '**** done rbd/test_librbd.sh 7-workload'
  - full_sequential:
    - rados:
        clients:
        - client.0
        objects: 500
        op_weights:
          delete: 10
          read: 45
          write: 45
        ops: 4000
        write_append_excl: false
  - print: '**** done rados/readwrite 5-workload'
  - full_sequential:
    - rados:
        clients:
        - client.0
        objects: 50
        op_weights:
          delete: 50
          read: 100
          rollback: 50
          snap_create: 50
          snap_remove: 50
          write: 100
        ops: 4000
        write_append_excl: false
  - print: '**** done rados/snaps-few-objects 5-workload'
  subset: 31/32
  suite: upgrade
  suite_branch: squid
  suite_path: /home/teuthworker/src/git.ceph.com_ceph_d0bf63dd179c05751e4be49477389aa290c73089/qa
  suite_relpath: qa
  suite_repo: https://git.ceph.com/ceph.git
  suite_sha1: d0bf63dd179c05751e4be49477389aa290c73089
  targets:
    smithi026.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFHKh6LkOrEvKFeTUma46CbsKIM0mlxXwveh6BrAQFNUGDo1UmKBqfZTTox3OECWzVshfAzYjdtdDavVYWfScO8=
    smithi164.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKnEg9Jg0W2+amaELbgkqJhxDUvGvyyW05vPjkzuWQi/XDd/P1V6iUoEq/lnFmC3ACliwmOaeAJvVJy48sxKi2Q=
  tasks:
  - install:
      branch: reef
      exclude_packages:
      - ceph-volume
  - cephadm:
      compiled_cephadm_branch: reef
      conf:
        osd:
          osd_class_default_list: '*'
          osd_class_load_list: '*'
      image: quay.ceph.io/ceph-ci/ceph:reef
  - cephadm.shell:
      mon.a:
      - ceph fs volume create foo
      - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
      - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
  - ceph.healthy: null
  - print: '**** upgrading first half of cluster, with stress ****'
  - parallel:
    - first-half-tasks
    - first-half-sequence
  - print: '**** done upgrading first half of cluster ****'
  - ceph.healthy: null
  - print: '**** applying stress + thrashing to mixed-version cluster ****'
  - parallel:
    - stress-tasks
  - ceph.healthy: null
  - print: '**** finishing upgrade ****'
  - parallel:
    - second-half-tasks
    - second-half-sequence
  - ceph.healthy: null
  teuthology:
    fragments_dropped: []
    meta: {}
    postmerge: []
  teuthology_branch: main
  teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17
  timestamp: 2024-04-19_02:08:09
  tube: smithi
  user: teuthology
  verbose: false
  worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885
2024-04-19T02:30:59.010 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_d0bf63dd179c05751e4be49477389aa290c73089/qa; will attempt to use it
2024-04-19T02:30:59.011 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_d0bf63dd179c05751e4be49477389aa290c73089/qa/tasks
2024-04-19T02:30:59.011 INFO:teuthology.run_tasks:Running task internal.check_packages...
2024-04-19T02:30:59.013 INFO:teuthology.task.internal:Checking packages...
2024-04-19T02:30:59.035 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '1a6a3e742fd81c6d70be70552499bff767652c32'
2024-04-19T02:30:59.035 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2024-04-19T02:30:59.035 INFO:teuthology.packaging:ref: None
2024-04-19T02:30:59.035 INFO:teuthology.packaging:tag: None
2024-04-19T02:30:59.035 INFO:teuthology.packaging:branch: squid
2024-04-19T02:30:59.035 INFO:teuthology.packaging:sha1: 1a6a3e742fd81c6d70be70552499bff767652c32
2024-04-19T02:30:59.035 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid
2024-04-19T02:30:59.249 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2401.gd0bf63dd
2024-04-19T02:30:59.251 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2024-04-19T02:30:59.257 INFO:teuthology.task.internal:no buildpackages task found
2024-04-19T02:30:59.257 INFO:teuthology.run_tasks:Running task internal.save_config...
2024-04-19T02:30:59.276 INFO:teuthology.task.internal:Saving configuration
2024-04-19T02:30:59.299 INFO:teuthology.run_tasks:Running task internal.check_lock...
2024-04-19T02:30:59.305 INFO:teuthology.task.internal.check_lock:Checking locks...
2024-04-19T02:30:59.328 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi026.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-19_02:08:09-upgrade-squid-distro-default-smithi/7663374', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-19 02:25:44.042201', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFHKh6LkOrEvKFeTUma46CbsKIM0mlxXwveh6BrAQFNUGDo1UmKBqfZTTox3OECWzVshfAzYjdtdDavVYWfScO8='}
2024-04-19T02:30:59.345 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi164.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-19_02:08:09-upgrade-squid-distro-default-smithi/7663374', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-19 02:25:44.043565', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKnEg9Jg0W2+amaELbgkqJhxDUvGvyyW05vPjkzuWQi/XDd/P1V6iUoEq/lnFmC3ACliwmOaeAJvVJy48sxKi2Q='}
2024-04-19T02:30:59.345 INFO:teuthology.run_tasks:Running task internal.add_remotes...
2024-04-19T02:30:59.351 INFO:teuthology.task.internal:roles: ubuntu@smithi026.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a']
2024-04-19T02:30:59.351 INFO:teuthology.task.internal:roles: ubuntu@smithi164.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b']
2024-04-19T02:30:59.351 INFO:teuthology.run_tasks:Running task console_log...
2024-04-19T02:30:59.419 DEBUG:teuthology.exit:Installing handler: Handler(exiter=<teuthology.exit.Exiter object at 0x7fa544127790>, func=<function ConsoleLog.start_logging.<locals>.kill_console_loggers at 0x7fa544ff40d0>, signals=[15])
2024-04-19T02:30:59.420 INFO:teuthology.run_tasks:Running task internal.connect...
2024-04-19T02:30:59.426 INFO:teuthology.task.internal:Opening connections...
2024-04-19T02:30:59.426 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi026.front.sepia.ceph.com
2024-04-19T02:30:59.428 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T02:30:59.537 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi164.front.sepia.ceph.com
2024-04-19T02:30:59.538 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T02:30:59.629 INFO:teuthology.run_tasks:Running task internal.push_inventory...
2024-04-19T02:30:59.636 DEBUG:teuthology.orchestra.run.smithi026:> uname -m
2024-04-19T02:30:59.655 INFO:teuthology.orchestra.run.smithi026.stdout:x86_64
2024-04-19T02:30:59.655 DEBUG:teuthology.orchestra.run.smithi026:> cat /etc/os-release
2024-04-19T02:30:59.714 INFO:teuthology.orchestra.run.smithi026.stdout:NAME="CentOS Stream"
2024-04-19T02:30:59.714 INFO:teuthology.orchestra.run.smithi026.stdout:VERSION="9"
2024-04-19T02:30:59.714 INFO:teuthology.orchestra.run.smithi026.stdout:ID="centos"
2024-04-19T02:30:59.714 INFO:teuthology.orchestra.run.smithi026.stdout:ID_LIKE="rhel fedora"
2024-04-19T02:30:59.714 INFO:teuthology.orchestra.run.smithi026.stdout:VERSION_ID="9"
2024-04-19T02:30:59.714 INFO:teuthology.orchestra.run.smithi026.stdout:PLATFORM_ID="platform:el9"
2024-04-19T02:30:59.714 INFO:teuthology.orchestra.run.smithi026.stdout:PRETTY_NAME="CentOS Stream 9"
2024-04-19T02:30:59.714 INFO:teuthology.orchestra.run.smithi026.stdout:ANSI_COLOR="0;31"
2024-04-19T02:30:59.714 INFO:teuthology.orchestra.run.smithi026.stdout:LOGO="fedora-logo-icon"
2024-04-19T02:30:59.714 INFO:teuthology.orchestra.run.smithi026.stdout:CPE_NAME="cpe:/o:centos:centos:9"
2024-04-19T02:30:59.714 INFO:teuthology.orchestra.run.smithi026.stdout:HOME_URL="https://centos.org/"
2024-04-19T02:30:59.715 INFO:teuthology.orchestra.run.smithi026.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/"
2024-04-19T02:30:59.715 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9"
2024-04-19T02:30:59.715 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream"
2024-04-19T02:30:59.715 INFO:teuthology.lock.ops:Updating smithi026.front.sepia.ceph.com on lock server
2024-04-19T02:30:59.736 DEBUG:teuthology.orchestra.run.smithi164:> uname -m
2024-04-19T02:30:59.753 INFO:teuthology.orchestra.run.smithi164.stdout:x86_64
2024-04-19T02:30:59.753 DEBUG:teuthology.orchestra.run.smithi164:> cat /etc/os-release
2024-04-19T02:30:59.810 INFO:teuthology.orchestra.run.smithi164.stdout:NAME="CentOS Stream"
2024-04-19T02:30:59.810 INFO:teuthology.orchestra.run.smithi164.stdout:VERSION="9"
2024-04-19T02:30:59.810 INFO:teuthology.orchestra.run.smithi164.stdout:ID="centos"
2024-04-19T02:30:59.810 INFO:teuthology.orchestra.run.smithi164.stdout:ID_LIKE="rhel fedora"
2024-04-19T02:30:59.810 INFO:teuthology.orchestra.run.smithi164.stdout:VERSION_ID="9"
2024-04-19T02:30:59.810 INFO:teuthology.orchestra.run.smithi164.stdout:PLATFORM_ID="platform:el9"
2024-04-19T02:30:59.810 INFO:teuthology.orchestra.run.smithi164.stdout:PRETTY_NAME="CentOS Stream 9"
2024-04-19T02:30:59.810 INFO:teuthology.orchestra.run.smithi164.stdout:ANSI_COLOR="0;31"
2024-04-19T02:30:59.811 INFO:teuthology.orchestra.run.smithi164.stdout:LOGO="fedora-logo-icon"
2024-04-19T02:30:59.811 INFO:teuthology.orchestra.run.smithi164.stdout:CPE_NAME="cpe:/o:centos:centos:9"
2024-04-19T02:30:59.811 INFO:teuthology.orchestra.run.smithi164.stdout:HOME_URL="https://centos.org/"
2024-04-19T02:30:59.811 INFO:teuthology.orchestra.run.smithi164.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/"
2024-04-19T02:30:59.811 INFO:teuthology.orchestra.run.smithi164.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9"
2024-04-19T02:30:59.811 INFO:teuthology.orchestra.run.smithi164.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream"
2024-04-19T02:30:59.811 INFO:teuthology.lock.ops:Updating smithi164.front.sepia.ceph.com on lock server
2024-04-19T02:30:59.836 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles...
2024-04-19T02:30:59.845 INFO:teuthology.run_tasks:Running task internal.check_conflict...
2024-04-19T02:30:59.850 INFO:teuthology.task.internal:Checking for old test directory...
2024-04-19T02:30:59.850 DEBUG:teuthology.orchestra.run.smithi026:> test '!' -e /home/ubuntu/cephtest
2024-04-19T02:30:59.853 DEBUG:teuthology.orchestra.run.smithi164:> test '!' -e /home/ubuntu/cephtest
2024-04-19T02:30:59.868 INFO:teuthology.run_tasks:Running task internal.check_ceph_data...
2024-04-19T02:30:59.873 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph...
2024-04-19T02:30:59.874 DEBUG:teuthology.orchestra.run.smithi026:> test -z $(ls -A /var/lib/ceph)
2024-04-19T02:30:59.910 DEBUG:teuthology.orchestra.run.smithi164:> test -z $(ls -A /var/lib/ceph)
2024-04-19T02:30:59.938 INFO:teuthology.run_tasks:Running task internal.vm_setup...
2024-04-19T02:30:59.989 INFO:teuthology.run_tasks:Running task kernel...
2024-04-19T02:31:00.004 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'}
2024-04-19T02:31:00.004 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300
2024-04-19T02:31:00.004 DEBUG:teuthology.orchestra.run.smithi026:> test -f /run/.containerenv -o -f /.dockerenv
2024-04-19T02:31:00.005 DEBUG:teuthology.orchestra.run.smithi164:> test -f /run/.containerenv -o -f /.dockerenv
2024-04-19T02:31:00.020 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T02:31:00.021 DEBUG:teuthology.orchestra.run.smithi164:> uname -r
2024-04-19T02:31:00.021 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T02:31:00.022 DEBUG:teuthology.orchestra.run.smithi026:> uname -r
2024-04-19T02:31:00.077 INFO:teuthology.orchestra.run.smithi164.stdout:5.14.0-361.el9.x86_64
2024-04-19T02:31:00.077 INFO:teuthology.task.kernel:Running kernel on smithi164: 5.14.0-361.el9.x86_64
2024-04-19T02:31:00.077 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum install -y kernel
2024-04-19T02:31:00.078 INFO:teuthology.orchestra.run.smithi026.stdout:5.14.0-361.el9.x86_64
2024-04-19T02:31:00.079 INFO:teuthology.task.kernel:Running kernel on smithi026: 5.14.0-361.el9.x86_64
2024-04-19T02:31:00.079 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum install -y kernel
2024-04-19T02:31:02.240 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS-9 - AppStream                             11 MB/s | 6.4 MB     00:00
2024-04-19T02:31:02.320 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS-9 - AppStream                             11 MB/s | 6.4 MB     00:00
2024-04-19T02:31:03.790 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS-9 - BaseOS                               4.5 MB/s | 2.0 MB     00:00
2024-04-19T02:31:03.928 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS-9 - BaseOS                               4.4 MB/s | 2.0 MB     00:00
2024-04-19T02:31:05.577 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - BaseOS                        6.2 MB/s | 8.0 MB     00:01
2024-04-19T02:31:05.722 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - BaseOS                        5.0 MB/s | 8.0 MB     00:01
2024-04-19T02:31:27.917 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - AppStream                     920 kB/s |  19 MB     00:21
2024-04-19T02:31:28.121 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - AppStream                     915 kB/s |  19 MB     00:21
2024-04-19T02:31:32.510 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - CRB                           8.4 MB/s | 6.2 MB     00:00
2024-04-19T02:31:33.001 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - CRB                           4.8 MB/s | 6.2 MB     00:01
2024-04-19T02:31:33.989 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - Extras packages                56 kB/s |  16 kB     00:00
2024-04-19T02:31:34.651 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - Extras packages                37 kB/s |  16 kB     00:00
2024-04-19T02:31:35.484 INFO:teuthology.orchestra.run.smithi164.stdout:Extra Packages for Enterprise Linux              16 MB/s |  21 MB     00:01
2024-04-19T02:31:35.836 INFO:teuthology.orchestra.run.smithi026.stdout:Extra Packages for Enterprise Linux              20 MB/s |  21 MB     00:01
2024-04-19T02:31:40.666 INFO:teuthology.orchestra.run.smithi164.stdout:lab-extras                                       14 kB/s | 1.7 kB     00:00
2024-04-19T02:31:41.128 INFO:teuthology.orchestra.run.smithi026.stdout:lab-extras                                       29 kB/s | 1.7 kB     00:00
2024-04-19T02:31:43.148 INFO:teuthology.orchestra.run.smithi164.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed.
2024-04-19T02:31:43.148 INFO:teuthology.orchestra.run.smithi164.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed.
2024-04-19T02:31:43.235 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved.
2024-04-19T02:31:43.240 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================
2024-04-19T02:31:43.240 INFO:teuthology.orchestra.run.smithi164.stdout: Package                Arch      Version                Repository        Size
2024-04-19T02:31:43.240 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================
2024-04-19T02:31:43.240 INFO:teuthology.orchestra.run.smithi164.stdout:Installing:
2024-04-19T02:31:43.240 INFO:teuthology.orchestra.run.smithi164.stdout: kernel                 x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     29 k
2024-04-19T02:31:43.240 INFO:teuthology.orchestra.run.smithi164.stdout:Installing dependencies:
2024-04-19T02:31:43.241 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-core            x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     16 M
2024-04-19T02:31:43.241 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-modules         x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     34 M
2024-04-19T02:31:43.241 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-modules-core    x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     27 M
2024-04-19T02:31:43.241 INFO:teuthology.orchestra.run.smithi164.stdout:
2024-04-19T02:31:43.241 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary
2024-04-19T02:31:43.241 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================
2024-04-19T02:31:43.241 INFO:teuthology.orchestra.run.smithi164.stdout:Install  4 Packages
2024-04-19T02:31:43.241 INFO:teuthology.orchestra.run.smithi164.stdout:
2024-04-19T02:31:43.241 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 78 M
2024-04-19T02:31:43.241 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 125 M
2024-04-19T02:31:43.241 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages:
2024-04-19T02:31:43.391 INFO:teuthology.orchestra.run.smithi164.stdout:(1/4): kernel-6.9.0-0.rc2.1.el10.x86_64.rpm     191 kB/s |  29 kB     00:00
2024-04-19T02:31:43.627 INFO:teuthology.orchestra.run.smithi026.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed.
2024-04-19T02:31:43.627 INFO:teuthology.orchestra.run.smithi026.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed.
2024-04-19T02:31:43.719 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved.
2024-04-19T02:31:43.729 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================
2024-04-19T02:31:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: Package                Arch      Version                Repository        Size
2024-04-19T02:31:43.729 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================
2024-04-19T02:31:43.729 INFO:teuthology.orchestra.run.smithi026.stdout:Installing:
2024-04-19T02:31:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: kernel                 x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     29 k
2024-04-19T02:31:43.729 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies:
2024-04-19T02:31:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: kernel-core            x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     16 M
2024-04-19T02:31:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: kernel-modules         x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     34 M
2024-04-19T02:31:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: kernel-modules-core    x86_64    6.9.0-0.rc2.1.el10     CentOS-BaseOS     27 M
2024-04-19T02:31:43.729 INFO:teuthology.orchestra.run.smithi026.stdout:
2024-04-19T02:31:43.729 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary
2024-04-19T02:31:43.729 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================
2024-04-19T02:31:43.730 INFO:teuthology.orchestra.run.smithi026.stdout:Install  4 Packages
2024-04-19T02:31:43.730 INFO:teuthology.orchestra.run.smithi026.stdout:
2024-04-19T02:31:43.730 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 78 M
2024-04-19T02:31:43.730 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 125 M
2024-04-19T02:31:43.730 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages:
2024-04-19T02:31:43.854 INFO:teuthology.orchestra.run.smithi026.stdout:(1/4): kernel-6.9.0-0.rc2.1.el10.x86_64.rpm     231 kB/s |  29 kB     00:00
2024-04-19T02:31:43.875 INFO:teuthology.orchestra.run.smithi164.stdout:(2/4): kernel-core-6.9.0-0.rc2.1.el10.x86_64.rp  26 MB/s |  16 MB     00:00
2024-04-19T02:31:44.742 INFO:teuthology.orchestra.run.smithi164.stdout:(3/4): kernel-modules-6.9.0-0.rc2.1.el10.x86_64  23 MB/s |  34 MB     00:01
2024-04-19T02:31:44.862 INFO:teuthology.orchestra.run.smithi026.stdout:(2/4): kernel-core-6.9.0-0.rc2.1.el10.x86_64.rp  14 MB/s |  16 MB     00:01
2024-04-19T02:31:44.992 INFO:teuthology.orchestra.run.smithi164.stdout:(4/4): kernel-modules-core-6.9.0-0.rc2.1.el10.x  17 MB/s |  27 MB     00:01
2024-04-19T02:31:44.995 INFO:teuthology.orchestra.run.smithi164.stdout:--------------------------------------------------------------------------------
2024-04-19T02:31:44.997 INFO:teuthology.orchestra.run.smithi164.stdout:Total                                            44 MB/s |  78 MB     00:01
2024-04-19T02:31:44.997 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check
2024-04-19T02:31:45.188 INFO:teuthology.orchestra.run.smithi026.stdout:(3/4): kernel-modules-core-6.9.0-0.rc2.1.el10.x  20 MB/s |  27 MB     00:01
2024-04-19T02:31:45.390 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded.
2024-04-19T02:31:45.390 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test
2024-04-19T02:31:45.571 INFO:teuthology.orchestra.run.smithi026.stdout:(4/4): kernel-modules-6.9.0-0.rc2.1.el10.x86_64  19 MB/s |  34 MB     00:01
2024-04-19T02:31:45.574 INFO:teuthology.orchestra.run.smithi026.stdout:--------------------------------------------------------------------------------
2024-04-19T02:31:45.574 INFO:teuthology.orchestra.run.smithi026.stdout:Total                                            42 MB/s |  78 MB     00:01
2024-04-19T02:31:45.575 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check
2024-04-19T02:31:46.036 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded.
2024-04-19T02:31:46.036 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test
2024-04-19T02:31:46.238 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded.
2024-04-19T02:31:46.238 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction
2024-04-19T02:31:46.916 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded.
2024-04-19T02:31:46.916 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction
2024-04-19T02:31:48.465 INFO:teuthology.orchestra.run.smithi164.stdout:  Preparing        :                                                        1/1
2024-04-19T02:31:49.146 INFO:teuthology.orchestra.run.smithi164.stdout:  Installing       : kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64          1/4
2024-04-19T02:31:49.233 INFO:teuthology.orchestra.run.smithi164.stdout:  Installing       : kernel-core-6.9.0-0.rc2.1.el10.x86_64                  2/4
2024-04-19T02:31:49.280 INFO:teuthology.orchestra.run.smithi026.stdout:  Preparing        :                                                        1/1
2024-04-19T02:31:49.939 INFO:teuthology.orchestra.run.smithi026.stdout:  Installing       : kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64          1/4
2024-04-19T02:31:50.029 INFO:teuthology.orchestra.run.smithi026.stdout:  Installing       : kernel-core-6.9.0-0.rc2.1.el10.x86_64                  2/4
2024-04-19T02:31:51.562 INFO:teuthology.orchestra.run.smithi164.stdout:  Running scriptlet: kernel-core-6.9.0-0.rc2.1.el10.x86_64                  2/4
2024-04-19T02:31:51.716 INFO:teuthology.orchestra.run.smithi164.stdout:  Installing       : kernel-modules-6.9.0-0.rc2.1.el10.x86_64               3/4
2024-04-19T02:31:52.456 INFO:teuthology.orchestra.run.smithi026.stdout:  Running scriptlet: kernel-core-6.9.0-0.rc2.1.el10.x86_64                  2/4
2024-04-19T02:31:52.594 INFO:teuthology.orchestra.run.smithi026.stdout:  Installing       : kernel-modules-6.9.0-0.rc2.1.el10.x86_64               3/4
2024-04-19T02:31:56.465 INFO:teuthology.orchestra.run.smithi164.stdout:  Running scriptlet: kernel-modules-6.9.0-0.rc2.1.el10.x86_64               3/4
2024-04-19T02:31:56.580 INFO:teuthology.orchestra.run.smithi164.stdout:  Installing       : kernel-6.9.0-0.rc2.1.el10.x86_64                       4/4
2024-04-19T02:31:57.770 INFO:teuthology.orchestra.run.smithi026.stdout:  Running scriptlet: kernel-modules-6.9.0-0.rc2.1.el10.x86_64               3/4
2024-04-19T02:31:57.808 INFO:teuthology.orchestra.run.smithi026.stdout:  Installing       : kernel-6.9.0-0.rc2.1.el10.x86_64                       4/4
2024-04-19T02:32:01.374 INFO:teuthology.orchestra.run.smithi164.stdout:  Running scriptlet: kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64          4/4
2024-04-19T02:32:03.011 INFO:teuthology.orchestra.run.smithi026.stdout:  Running scriptlet: kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64          4/4
2024-04-19T02:32:31.265 INFO:teuthology.orchestra.run.smithi164.stdout:  Running scriptlet: kernel-core-6.9.0-0.rc2.1.el10.x86_64                  4/4
2024-04-19T02:32:31.266 INFO:teuthology.orchestra.run.smithi164.stdout:
2024-04-19T02:32:31.266 INFO:teuthology.orchestra.run.smithi164.stdout:gzip: /lib/modules/6.9.0-0.rc2.1.el10.x86_64/symvers.xz: not in gzip format
2024-04-19T02:32:31.266 INFO:teuthology.orchestra.run.smithi164.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y
2024-04-19T02:32:31.266 INFO:teuthology.orchestra.run.smithi164.stdout:
2024-04-19T02:32:31.308 INFO:teuthology.orchestra.run.smithi164.stdout:  Running scriptlet: kernel-modules-6.9.0-0.rc2.1.el10.x86_64               4/4
2024-04-19T02:32:31.708 INFO:teuthology.orchestra.run.smithi164.stdout:  Running scriptlet: kernel-6.9.0-0.rc2.1.el10.x86_64                       4/4
2024-04-19T02:32:31.708 INFO:teuthology.orchestra.run.smithi164.stdout:  Verifying        : kernel-6.9.0-0.rc2.1.el10.x86_64                       1/4
2024-04-19T02:32:31.708 INFO:teuthology.orchestra.run.smithi164.stdout:  Verifying        : kernel-core-6.9.0-0.rc2.1.el10.x86_64                  2/4
2024-04-19T02:32:31.708 INFO:teuthology.orchestra.run.smithi164.stdout:  Verifying        : kernel-modules-6.9.0-0.rc2.1.el10.x86_64               3/4
2024-04-19T02:32:32.122 INFO:teuthology.orchestra.run.smithi164.stdout:  Verifying        : kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64          4/4
2024-04-19T02:32:32.122 INFO:teuthology.orchestra.run.smithi164.stdout:
2024-04-19T02:32:32.122 INFO:teuthology.orchestra.run.smithi164.stdout:Installed:
2024-04-19T02:32:32.122 INFO:teuthology.orchestra.run.smithi164.stdout:  kernel-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:32.122 INFO:teuthology.orchestra.run.smithi164.stdout:  kernel-core-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:32.122 INFO:teuthology.orchestra.run.smithi164.stdout:  kernel-modules-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:32.122 INFO:teuthology.orchestra.run.smithi164.stdout:  kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:32.122 INFO:teuthology.orchestra.run.smithi164.stdout:
2024-04-19T02:32:32.123 INFO:teuthology.orchestra.run.smithi164.stdout:Complete!
2024-04-19T02:32:32.480 DEBUG:teuthology.orchestra.run.smithi164:> rpm -q kernel | sort -rV | head -n 1
2024-04-19T02:32:32.577 INFO:teuthology.orchestra.run.smithi164.stdout:kernel-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:32.577 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:32.577 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:32.577 INFO:teuthology.task.kernel:Skipping firmware on distro kernel
2024-04-19T02:32:32.577 DEBUG:teuthology.task.kernel:src is distro, skipping download
2024-04-19T02:32:32.577 INFO:teuthology.task.kernel:Installing distro kernel on mon.b...
2024-04-19T02:32:32.578 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi164.front.sepia.ceph.com, path=None, version=distro)
2024-04-19T02:32:32.578 DEBUG:teuthology.orchestra.run.smithi164:> rpm -q kernel | sort -rV | head -n 1
2024-04-19T02:32:32.656 INFO:teuthology.orchestra.run.smithi164.stdout:kernel-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:32.657 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:32.657 DEBUG:teuthology.orchestra.run.smithi164:> sudo rpm -qi grub2-tools
2024-04-19T02:32:32.691 INFO:teuthology.orchestra.run.smithi164.stdout:Name        : grub2-tools
2024-04-19T02:32:32.691 INFO:teuthology.orchestra.run.smithi164.stdout:Epoch       : 1
2024-04-19T02:32:32.691 INFO:teuthology.orchestra.run.smithi164.stdout:Version     : 2.06
2024-04-19T02:32:32.691 INFO:teuthology.orchestra.run.smithi164.stdout:Release     : 68.el9
2024-04-19T02:32:32.691 INFO:teuthology.orchestra.run.smithi164.stdout:Architecture: x86_64
2024-04-19T02:32:32.691 INFO:teuthology.orchestra.run.smithi164.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC
2024-04-19T02:32:32.691 INFO:teuthology.orchestra.run.smithi164.stdout:Group       : Unspecified
2024-04-19T02:32:32.691 INFO:teuthology.orchestra.run.smithi164.stdout:Size        : 8264254
2024-04-19T02:32:32.691 INFO:teuthology.orchestra.run.smithi164.stdout:License     : GPLv3+
2024-04-19T02:32:32.691 INFO:teuthology.orchestra.run.smithi164.stdout:Signature   : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:Source RPM  : grub2-2.06-68.el9.src.rpm
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:Build Date  : Wed 02 Aug 2023 03:15:48 PM UTC
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:Build Host  : x86-05.stream.rdu2.redhat.com
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:Packager    : builder@centos.org
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:Vendor      : CentOS
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:URL         : http://www.gnu.org/software/grub/
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:Summary     : Support tools for GRUB.
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:Description :
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:customizable bootloader with modular architecture.  It supports a rich
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:variety of kernel formats, file systems, computer architectures and
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:hardware devices.
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:
2024-04-19T02:32:32.692 INFO:teuthology.orchestra.run.smithi164.stdout:This subpackage provides tools for support of all platforms.
2024-04-19T02:32:32.693 INFO:teuthology.task.kernel:Updating Grub Version: grub2
2024-04-19T02:32:32.693 INFO:teuthology.task.kernel:Updating grub on smithi164 to boot 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:32.694 DEBUG:teuthology.orchestra.run.smithi164:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg
2024-04-19T02:32:33.323 INFO:teuthology.orchestra.run.smithi164.stderr:Generating grub configuration file ...
2024-04-19T02:32:34.248 INFO:teuthology.orchestra.run.smithi026.stdout:  Running scriptlet: kernel-core-6.9.0-0.rc2.1.el10.x86_64                  4/4
2024-04-19T02:32:34.248 INFO:teuthology.orchestra.run.smithi026.stdout:
2024-04-19T02:32:34.248 INFO:teuthology.orchestra.run.smithi026.stdout:gzip: /lib/modules/6.9.0-0.rc2.1.el10.x86_64/symvers.xz: not in gzip format
2024-04-19T02:32:34.248 INFO:teuthology.orchestra.run.smithi026.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y
2024-04-19T02:32:34.248 INFO:teuthology.orchestra.run.smithi026.stdout:
2024-04-19T02:32:34.297 INFO:teuthology.orchestra.run.smithi026.stdout:  Running scriptlet: kernel-modules-6.9.0-0.rc2.1.el10.x86_64               4/4
2024-04-19T02:32:34.713 INFO:teuthology.orchestra.run.smithi164.stderr:Adding boot menu entry for UEFI Firmware Settings ...
2024-04-19T02:32:34.727 INFO:teuthology.orchestra.run.smithi026.stdout:  Running scriptlet: kernel-6.9.0-0.rc2.1.el10.x86_64                       4/4
2024-04-19T02:32:34.727 INFO:teuthology.orchestra.run.smithi026.stdout:  Verifying        : kernel-6.9.0-0.rc2.1.el10.x86_64                       1/4
2024-04-19T02:32:34.727 INFO:teuthology.orchestra.run.smithi026.stdout:  Verifying        : kernel-core-6.9.0-0.rc2.1.el10.x86_64                  2/4
2024-04-19T02:32:34.727 INFO:teuthology.orchestra.run.smithi026.stdout:  Verifying        : kernel-modules-6.9.0-0.rc2.1.el10.x86_64               3/4
2024-04-19T02:32:34.744 INFO:teuthology.orchestra.run.smithi164.stderr:done
2024-04-19T02:32:34.747 DEBUG:teuthology.orchestra.run.smithi164:> mktemp
2024-04-19T02:32:34.762 INFO:teuthology.orchestra.run.smithi164.stdout:/tmp/tmp.NdcmtLfyak
2024-04-19T02:32:34.763 DEBUG:teuthology.orchestra.run.smithi164:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.NdcmtLfyak
2024-04-19T02:32:34.831 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 0666 /tmp/tmp.NdcmtLfyak
2024-04-19T02:32:34.967 DEBUG:teuthology.orchestra.remote:smithi164:/tmp/tmp.NdcmtLfyak is 6KB
2024-04-19T02:32:35.015 DEBUG:teuthology.orchestra.run.smithi164:> rm -fr /tmp/tmp.NdcmtLfyak
2024-04-19T02:32:35.029 DEBUG:teuthology.orchestra.run.smithi164:> sudo /bin/ls /boot/loader/entries || true
2024-04-19T02:32:35.094 INFO:teuthology.orchestra.run.smithi164.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf
2024-04-19T02:32:35.094 INFO:teuthology.orchestra.run.smithi164.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf
2024-04-19T02:32:35.094 INFO:teuthology.orchestra.run.smithi164.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf
2024-04-19T02:32:35.094 INFO:teuthology.orchestra.run.smithi164.stdout:15949e3c88704300882482eddfd7baae-6.9.0-0.rc2.1.el10.x86_64.conf
2024-04-19T02:32:35.096 DEBUG:teuthology.orchestra.run.smithi164:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:35.161 INFO:teuthology.orchestra.run.smithi026.stdout:  Verifying        : kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64          4/4
2024-04-19T02:32:35.161 INFO:teuthology.orchestra.run.smithi026.stdout:
2024-04-19T02:32:35.161 INFO:teuthology.orchestra.run.smithi026.stdout:Installed:
2024-04-19T02:32:35.161 INFO:teuthology.orchestra.run.smithi026.stdout:  kernel-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:35.161 INFO:teuthology.orchestra.run.smithi026.stdout:  kernel-core-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:35.161 INFO:teuthology.orchestra.run.smithi026.stdout:  kernel-modules-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:35.162 INFO:teuthology.orchestra.run.smithi026.stdout:  kernel-modules-core-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:35.162 INFO:teuthology.orchestra.run.smithi026.stdout:
2024-04-19T02:32:35.162 INFO:teuthology.orchestra.run.smithi026.stdout:Complete!
2024-04-19T02:32:35.245 DEBUG:teuthology.orchestra.run.smithi164:> sudo shutdown -r now
2024-04-19T02:32:35.487 DEBUG:teuthology.orchestra.run.smithi026:> rpm -q kernel | sort -rV | head -n 1
2024-04-19T02:32:35.593 INFO:teuthology.orchestra.run.smithi026.stdout:kernel-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:35.593 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:35.594 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:35.594 INFO:teuthology.task.kernel:Skipping firmware on distro kernel
2024-04-19T02:32:35.594 DEBUG:teuthology.task.kernel:src is distro, skipping download
2024-04-19T02:32:35.594 INFO:teuthology.task.kernel:Installing distro kernel on mon.a...
2024-04-19T02:32:35.594 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi026.front.sepia.ceph.com, path=None, version=distro)
2024-04-19T02:32:35.594 DEBUG:teuthology.orchestra.run.smithi026:> rpm -q kernel | sort -rV | head -n 1
2024-04-19T02:32:35.676 INFO:teuthology.orchestra.run.smithi026.stdout:kernel-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:35.677 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:35.677 DEBUG:teuthology.orchestra.run.smithi026:> sudo rpm -qi grub2-tools
2024-04-19T02:32:35.715 INFO:teuthology.orchestra.run.smithi026.stdout:Name        : grub2-tools
2024-04-19T02:32:35.715 INFO:teuthology.orchestra.run.smithi026.stdout:Epoch       : 1
2024-04-19T02:32:35.715 INFO:teuthology.orchestra.run.smithi026.stdout:Version     : 2.06
2024-04-19T02:32:35.715 INFO:teuthology.orchestra.run.smithi026.stdout:Release     : 68.el9
2024-04-19T02:32:35.715 INFO:teuthology.orchestra.run.smithi026.stdout:Architecture: x86_64
2024-04-19T02:32:35.715 INFO:teuthology.orchestra.run.smithi026.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC
2024-04-19T02:32:35.715 INFO:teuthology.orchestra.run.smithi026.stdout:Group       : Unspecified
2024-04-19T02:32:35.715 INFO:teuthology.orchestra.run.smithi026.stdout:Size        : 8264254
2024-04-19T02:32:35.715 INFO:teuthology.orchestra.run.smithi026.stdout:License     : GPLv3+
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:Signature   : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:Source RPM  : grub2-2.06-68.el9.src.rpm
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:Build Date  : Wed 02 Aug 2023 03:15:48 PM UTC
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:Build Host  : x86-05.stream.rdu2.redhat.com
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:Packager    : builder@centos.org
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:Vendor      : CentOS
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:URL         : http://www.gnu.org/software/grub/
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:Summary     : Support tools for GRUB.
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:Description :
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:customizable bootloader with modular architecture.  It supports a rich
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:variety of kernel formats, file systems, computer architectures and
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:hardware devices.
2024-04-19T02:32:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:
2024-04-19T02:32:35.717 INFO:teuthology.orchestra.run.smithi026.stdout:This subpackage provides tools for support of all platforms.
2024-04-19T02:32:35.717 INFO:teuthology.task.kernel:Updating Grub Version: grub2
2024-04-19T02:32:35.717 INFO:teuthology.task.kernel:Updating grub on smithi026 to boot 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:35.717 DEBUG:teuthology.orchestra.run.smithi026:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg
2024-04-19T02:32:36.336 INFO:teuthology.orchestra.run.smithi026.stderr:Generating grub configuration file ...
2024-04-19T02:32:37.744 INFO:teuthology.orchestra.run.smithi026.stderr:Adding boot menu entry for UEFI Firmware Settings ...
2024-04-19T02:32:37.781 INFO:teuthology.orchestra.run.smithi026.stderr:done
2024-04-19T02:32:37.784 DEBUG:teuthology.orchestra.run.smithi026:> mktemp
2024-04-19T02:32:37.799 INFO:teuthology.orchestra.run.smithi026.stdout:/tmp/tmp.KB81O0Hhd9
2024-04-19T02:32:37.799 DEBUG:teuthology.orchestra.run.smithi026:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.KB81O0Hhd9
2024-04-19T02:32:37.871 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod 0666 /tmp/tmp.KB81O0Hhd9
2024-04-19T02:32:38.019 DEBUG:teuthology.orchestra.remote:smithi026:/tmp/tmp.KB81O0Hhd9 is 6KB
2024-04-19T02:32:38.039 DEBUG:teuthology.orchestra.run.smithi026:> rm -fr /tmp/tmp.KB81O0Hhd9
2024-04-19T02:32:38.054 DEBUG:teuthology.orchestra.run.smithi026:> sudo /bin/ls /boot/loader/entries || true
2024-04-19T02:32:38.123 INFO:teuthology.orchestra.run.smithi026.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf
2024-04-19T02:32:38.123 INFO:teuthology.orchestra.run.smithi026.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf
2024-04-19T02:32:38.123 INFO:teuthology.orchestra.run.smithi026.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf
2024-04-19T02:32:38.123 INFO:teuthology.orchestra.run.smithi026.stdout:15949e3c88704300882482eddfd7baae-6.9.0-0.rc2.1.el10.x86_64.conf
2024-04-19T02:32:38.124 DEBUG:teuthology.orchestra.run.smithi026:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:32:38.283 DEBUG:teuthology.orchestra.run.smithi026:> sudo shutdown -r now
2024-04-19T02:33:05.274 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version...
2024-04-19T02:33:05.275 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com'
2024-04-19T02:33:05.276 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T02:33:08.288 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version...
2024-04-19T02:33:08.289 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi026.front.sepia.ceph.com'
2024-04-19T02:33:08.289 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T02:33:23.743 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.164
2024-04-19T02:33:26.818 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.26
2024-04-19T02:33:32.748 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com'
2024-04-19T02:33:32.751 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T02:33:35.806 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.164
2024-04-19T02:33:35.819 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi026.front.sepia.ceph.com'
2024-04-19T02:33:35.819 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T02:33:47.818 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com'
2024-04-19T02:33:47.819 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T02:33:54.206 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.26
2024-04-19T02:34:06.218 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi026.front.sepia.ceph.com'
2024-04-19T02:34:06.219 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T02:34:06.639 DEBUG:teuthology.orchestra.run.smithi026:> true
2024-04-19T02:34:07.038 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi026.front.sepia.ceph.com'
2024-04-19T02:34:07.038 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "6.9.0-0.rc2.1.el10.x86_64"...
2024-04-19T02:34:07.039 DEBUG:teuthology.orchestra.run.smithi026:> uname -r
2024-04-19T02:34:07.053 INFO:teuthology.orchestra.run.smithi026.stdout:6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:34:07.054 DEBUG:teuthology.task.kernel:current kernel version is 6.9.0-0.rc2.1.el10.x86_64 vs 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:34:07.054 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install
2024-04-19T02:34:07.054 DEBUG:teuthology.task.kernel:Distro of this test job: centos
2024-04-19T02:34:08.055 INFO:teuthology.task.kernel:Enabling kdb on mon.a...
2024-04-19T02:34:08.055 DEBUG:teuthology.orchestra.run.smithi026:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2024-04-19T02:34:08.127 INFO:teuthology.orchestra.run.smithi026.stdout:ttyS1
2024-04-19T02:34:08.178 DEBUG:teuthology.parallel:result is None
2024-04-19T02:34:47.821 DEBUG:teuthology.orchestra.remote:timed out
2024-04-19T02:35:02.823 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com'
2024-04-19T02:35:02.824 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-19T02:35:03.147 DEBUG:teuthology.orchestra.run.smithi164:> true
2024-04-19T02:35:03.592 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi164.front.sepia.ceph.com'
2024-04-19T02:35:03.592 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "6.9.0-0.rc2.1.el10.x86_64"...
2024-04-19T02:35:03.593 DEBUG:teuthology.orchestra.run.smithi164:> uname -r
2024-04-19T02:35:03.609 INFO:teuthology.orchestra.run.smithi164.stdout:6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:35:03.609 DEBUG:teuthology.task.kernel:current kernel version is 6.9.0-0.rc2.1.el10.x86_64 vs 6.9.0-0.rc2.1.el10.x86_64
2024-04-19T02:35:03.610 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install
2024-04-19T02:35:03.610 DEBUG:teuthology.task.kernel:Distro of this test job: centos
2024-04-19T02:35:04.611 INFO:teuthology.task.kernel:Enabling kdb on mon.b...
2024-04-19T02:35:04.611 DEBUG:teuthology.orchestra.run.smithi164:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2024-04-19T02:35:04.694 INFO:teuthology.orchestra.run.smithi164.stdout:ttyS1
2024-04-19T02:35:04.758 DEBUG:teuthology.parallel:result is None
2024-04-19T02:35:04.758 INFO:teuthology.run_tasks:Running task internal.base...
2024-04-19T02:35:04.765 INFO:teuthology.task.internal:Creating test directory...
2024-04-19T02:35:04.766 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2024-04-19T02:35:04.769 DEBUG:teuthology.orchestra.run.smithi164:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2024-04-19T02:35:04.788 INFO:teuthology.run_tasks:Running task internal.archive_upload...
2024-04-19T02:35:04.795 INFO:teuthology.run_tasks:Running task internal.archive...
2024-04-19T02:35:04.802 INFO:teuthology.task.internal:Creating archive directory...
2024-04-19T02:35:04.803 DEBUG:teuthology.orchestra.run.smithi026:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2024-04-19T02:35:04.830 DEBUG:teuthology.orchestra.run.smithi164:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2024-04-19T02:35:04.867 INFO:teuthology.run_tasks:Running task internal.coredump...
2024-04-19T02:35:04.874 INFO:teuthology.task.internal:Enabling coredump saving...
2024-04-19T02:35:04.874 DEBUG:teuthology.orchestra.run.smithi026:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2024-04-19T02:35:04.894 DEBUG:teuthology.orchestra.run.smithi164:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2024-04-19T02:35:04.930 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-19T02:35:04.938 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-19T02:35:04.945 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-19T02:35:04.953 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-19T02:35:04.955 INFO:teuthology.run_tasks:Running task internal.sudo...
2024-04-19T02:35:04.963 INFO:teuthology.task.internal:Configuring sudo...
2024-04-19T02:35:04.964 DEBUG:teuthology.orchestra.run.smithi026:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2024-04-19T02:35:04.990 DEBUG:teuthology.orchestra.run.smithi164:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2024-04-19T02:35:05.030 INFO:teuthology.run_tasks:Running task internal.syslog...
2024-04-19T02:35:05.040 INFO:teuthology.task.internal.syslog:Starting syslog monitoring...
2024-04-19T02:35:05.040 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2024-04-19T02:35:05.065 DEBUG:teuthology.orchestra.run.smithi164:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2024-04-19T02:35:05.087 DEBUG:teuthology.orchestra.run.smithi026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-19T02:35:05.167 DEBUG:teuthology.orchestra.run.smithi026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-19T02:35:05.235 DEBUG:teuthology.orchestra.run.smithi026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-19T02:35:05.278 DEBUG:teuthology.orchestra.run.smithi026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-19T02:35:05.328 DEBUG:teuthology.orchestra.run.smithi026:> set -ex
2024-04-19T02:35:05.328 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2024-04-19T02:35:05.401 DEBUG:teuthology.orchestra.run.smithi164:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-19T02:35:05.448 DEBUG:teuthology.orchestra.run.smithi164:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-19T02:35:05.517 DEBUG:teuthology.orchestra.run.smithi164:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-19T02:35:05.563 DEBUG:teuthology.orchestra.run.smithi164:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-19T02:35:05.608 DEBUG:teuthology.orchestra.run.smithi164:> set -ex
2024-04-19T02:35:05.608 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2024-04-19T02:35:05.678 DEBUG:teuthology.orchestra.run.smithi026:> sudo service rsyslog restart
2024-04-19T02:35:05.681 DEBUG:teuthology.orchestra.run.smithi164:> sudo service rsyslog restart
2024-04-19T02:35:05.737 INFO:teuthology.orchestra.run.smithi026.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-19T02:35:05.771 INFO:teuthology.orchestra.run.smithi164.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-19T02:35:06.251 INFO:teuthology.run_tasks:Running task internal.timer...
2024-04-19T02:35:06.260 INFO:teuthology.task.internal:Starting timer...
2024-04-19T02:35:06.260 INFO:teuthology.run_tasks:Running task pcp...
2024-04-19T02:35:06.272 INFO:teuthology.run_tasks:Running task selinux...
2024-04-19T02:35:06.281 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}
2024-04-19T02:35:06.282 DEBUG:teuthology.orchestra.run.smithi026:> sudo service auditd rotate
2024-04-19T02:35:06.355 INFO:teuthology.orchestra.run.smithi026.stdout:Rotating logs:
2024-04-19T02:35:06.357 DEBUG:teuthology.orchestra.run.smithi164:> sudo service auditd rotate
2024-04-19T02:35:06.426 INFO:teuthology.orchestra.run.smithi164.stdout:Rotating logs:
2024-04-19T02:35:06.429 DEBUG:teuthology.task.selinux:Getting current SELinux state
2024-04-19T02:35:06.429 DEBUG:teuthology.orchestra.run.smithi026:> /usr/sbin/getenforce
2024-04-19T02:35:06.450 INFO:teuthology.orchestra.run.smithi026.stdout:Permissive
2024-04-19T02:35:06.450 DEBUG:teuthology.orchestra.run.smithi164:> /usr/sbin/getenforce
2024-04-19T02:35:06.469 INFO:teuthology.orchestra.run.smithi164.stdout:Permissive
2024-04-19T02:35:06.470 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi026.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi164.front.sepia.ceph.com': 'permissive'}
2024-04-19T02:35:06.470 DEBUG:teuthology.orchestra.run.smithi026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0
2024-04-19T02:35:06.519 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T02:35:06.520 DEBUG:teuthology.orchestra.run.smithi164:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0
2024-04-19T02:35:06.549 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T02:35:06.550 INFO:teuthology.task.selinux:Putting SELinux into permissive mode
2024-04-19T02:35:06.550 DEBUG:teuthology.orchestra.run.smithi026:> sudo /usr/sbin/setenforce permissive
2024-04-19T02:35:06.593 DEBUG:teuthology.orchestra.run.smithi164:> sudo /usr/sbin/setenforce permissive
2024-04-19T02:35:06.630 INFO:teuthology.run_tasks:Running task ansible.cephlab...
2024-04-19T02:35:06.640 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main
2024-04-19T02:35:06.658 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}]
2024-04-19T02:35:06.659 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi026.front.sepia.ceph.com,smithi164.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml
2024-04-19T02:38:51.812 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-19_02:08:09-upgrade-squid-distro-default-smithi/7663374/ansible_failures.yaml
2024-04-19T02:38:51.814 ERROR:teuthology.run_tasks:Saw exception from tasks.
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 109, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 508, in begin
    super(CephLab, self).begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 342, in begin
    self.execute_playbook()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 374, in execute_playbook
    self._handle_failure(command, status)
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 402, in _handle_failure
    raise AnsibleFailedError(failures)
teuthology.exceptions.AnsibleFailedError: smithi164.front.sepia.ceph.com:  _ansible_no_log: null  changed: false  failures: []  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      cacheonly: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - dnf-utils      - sysstat      - libedit      - boost-thread      - xfsprogs      - gdisk      - parted      - libgcrypt      - fuse-libs      - openssl      - libuuid      - podman      - attr      - lsof      - gettext      - bc      - xfsdump      - blktrace      - usbredir      - valgrind      - nfs-utils      - ncurses-devel      - gcc      - git      - genisoimage      - qemu-img      - qemu-kvm-core      - qemu-kvm-block-rbd      - libacl-devel      - autoconf      - gdb      - iozone      nobest: false      releasever: null      security: false      skip_broken: false      sslverify: true      state: present      update_cache: false      update_only: false      validate_certs: true  msg: "Depsolve Error occurred: \n Problem: problem with installed package annobin-12.12-1.el9.x86_64\n\    \  - package annobin-12.12-1.el9.x86_64 from @System requires (gcc >= 11 with\    \ gcc < 12), but none of the providers can be installed\n  - package annobin-12.09-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.12-1.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - package\    \ annobin-12.28-1.el9.x86_64 from appstream requires (gcc >= 11 with gcc < 12),\    \ but none of the providers can be installed\n  - package annobin-12.31-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.31-2.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - cannot\    \ install both gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream and gcc-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both gcc-11.3.1-4.4.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.1.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-2.3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - package gcc-c++-14.0.1-0.15.el10.x86_64\    \ from CentOS-AppStream requires gcc = 14.0.1-0.15.el10, but none of the providers\    \ can be installed\n  - problem with installed package gcc-c++-11.4.1-2.1.el9.x86_64\n\    \  - package gcc-c++-11.4.1-2.1.el9.x86_64 from appstream requires libstdc++ =\    \ 11.4.1-2.1.el9, but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.1.el9.x86_64\    \ from @System requires libstdc++ = 11.4.1-2.1.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.3.1-4.4.el9.x86_64 from appstream requires\    \ libstdc++ = 11.3.1-4.4.el9, but none of the providers can be installed\n  -\    \ package gcc-c++-11.4.1-2.3.el9.x86_64 from appstream requires libstdc++ = 11.4.1-2.3.el9,\    \ but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.el9.x86_64\    \ from appstream requires libstdc++ = 11.4.1-2.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.4.1-3.el9.x86_64 from appstream requires\    \ libstdc++ = 11.4.1-3.el9, but none of the providers can be installed\n  - libstdc++-11.4.1-2.1.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.3.1-4.4.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - cannot install\    \ both libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS and libstdc++-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both libstdc++-11.3.1-4.4.el9.x86_64 from baseos\    \ and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot install\    \ both libstdc++-11.4.1-2.1.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-2.3.el9.x86_64\    \ from baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot\    \ install both libstdc++-11.4.1-2.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-3.el9.x86_64 from\    \ baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - package\    \ python3-hawkey-0.73.0-1.el10.x86_64 from CentOS-BaseOS requires libstdc++.so.6(GLIBCXX_3.4.30)(64bit),\    \ but none of the providers can be installed\n  - package python3-dnf-4.19.0-2.el10.noarch\    \ from CentOS-BaseOS requires python3-hawkey >= 0.73.0, but none of the providers\    \ can be installed\n  - package yum-utils-4.5.0-1.el10.noarch from CentOS-BaseOS\    \ requires python3-dnf >= 4.19.0, but none of the providers can be installed\n\    \  - cannot install the best candidate for the job"  rc: 1  results: []smithi026.front.sepia.ceph.com:  _ansible_no_log: null  changed: false  failures: []  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      cacheonly: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - dnf-utils      - sysstat      - libedit      - boost-thread      - xfsprogs      - gdisk      - parted      - libgcrypt      - fuse-libs      - openssl      - libuuid      - podman      - attr      - lsof      - gettext      - bc      - xfsdump      - blktrace      - usbredir      - valgrind      - nfs-utils      - ncurses-devel      - gcc      - git      - genisoimage      - qemu-img      - qemu-kvm-core      - qemu-kvm-block-rbd      - libacl-devel      - autoconf      - gdb      - iozone      nobest: false      releasever: null      security: false      skip_broken: false      sslverify: true      state: present      update_cache: false      update_only: false      validate_certs: true  msg: "Depsolve Error occurred: \n Problem: problem with installed package annobin-12.12-1.el9.x86_64\n\    \  - package annobin-12.12-1.el9.x86_64 from @System requires (gcc >= 11 with\    \ gcc < 12), but none of the providers can be installed\n  - package annobin-12.09-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.12-1.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - package\    \ annobin-12.28-1.el9.x86_64 from appstream requires (gcc >= 11 with gcc < 12),\    \ but none of the providers can be installed\n  - package annobin-12.31-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.31-2.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - cannot\    \ install both gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream and gcc-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both gcc-11.3.1-4.4.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.1.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-2.3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - package gcc-c++-14.0.1-0.15.el10.x86_64\    \ from CentOS-AppStream requires gcc = 14.0.1-0.15.el10, but none of the providers\    \ can be installed\n  - problem with installed package gcc-c++-11.4.1-2.1.el9.x86_64\n\    \  - package gcc-c++-11.4.1-2.1.el9.x86_64 from appstream requires libstdc++ =\    \ 11.4.1-2.1.el9, but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.1.el9.x86_64\    \ from @System requires libstdc++ = 11.4.1-2.1.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.3.1-4.4.el9.x86_64 from appstream requires\    \ libstdc++ = 11.3.1-4.4.el9, but none of the providers can be installed\n  -\    \ package gcc-c++-11.4.1-2.3.el9.x86_64 from appstream requires libstdc++ = 11.4.1-2.3.el9,\    \ but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.el9.x86_64\    \ from appstream requires libstdc++ = 11.4.1-2.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.4.1-3.el9.x86_64 from appstream requires\    \ libstdc++ = 11.4.1-3.el9, but none of the providers can be installed\n  - libstdc++-11.4.1-2.1.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.3.1-4.4.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - cannot install\    \ both libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS and libstdc++-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both libstdc++-11.3.1-4.4.el9.x86_64 from baseos\    \ and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot install\    \ both libstdc++-11.4.1-2.1.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-2.3.el9.x86_64\    \ from baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot\    \ install both libstdc++-11.4.1-2.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-3.el9.x86_64 from\    \ baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - package\    \ python3-hawkey-0.73.0-1.el10.x86_64 from CentOS-BaseOS requires libstdc++.so.6(GLIBCXX_3.4.30)(64bit),\    \ but none of the providers can be installed\n  - package python3-dnf-4.19.0-2.el10.noarch\    \ from CentOS-BaseOS requires python3-hawkey >= 0.73.0, but none of the providers\    \ can be installed\n  - package yum-utils-4.5.0-1.el10.noarch from CentOS-BaseOS\    \ requires python3-dnf >= 4.19.0, but none of the providers can be installed\n\    \  - cannot install the best candidate for the job"  rc: 1  results: []
2024-04-19T02:38:52.051 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=c26ad245b4884e8990a5065af425dbc8
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 109, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 508, in begin
    super(CephLab, self).begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 342, in begin
    self.execute_playbook()
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 374, in execute_playbook
    self._handle_failure(command, status)
  File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/task/ansible.py", line 402, in _handle_failure
    raise AnsibleFailedError(failures)
teuthology.exceptions.AnsibleFailedError: smithi164.front.sepia.ceph.com:  _ansible_no_log: null  changed: false  failures: []  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      cacheonly: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - dnf-utils      - sysstat      - libedit      - boost-thread      - xfsprogs      - gdisk      - parted      - libgcrypt      - fuse-libs      - openssl      - libuuid      - podman      - attr      - lsof      - gettext      - bc      - xfsdump      - blktrace      - usbredir      - valgrind      - nfs-utils      - ncurses-devel      - gcc      - git      - genisoimage      - qemu-img      - qemu-kvm-core      - qemu-kvm-block-rbd      - libacl-devel      - autoconf      - gdb      - iozone      nobest: false      releasever: null      security: false      skip_broken: false      sslverify: true      state: present      update_cache: false      update_only: false      validate_certs: true  msg: "Depsolve Error occurred: \n Problem: problem with installed package annobin-12.12-1.el9.x86_64\n\    \  - package annobin-12.12-1.el9.x86_64 from @System requires (gcc >= 11 with\    \ gcc < 12), but none of the providers can be installed\n  - package annobin-12.09-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.12-1.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - package\    \ annobin-12.28-1.el9.x86_64 from appstream requires (gcc >= 11 with gcc < 12),\    \ but none of the providers can be installed\n  - package annobin-12.31-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.31-2.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - cannot\    \ install both gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream and gcc-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both gcc-11.3.1-4.4.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.1.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-2.3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - package gcc-c++-14.0.1-0.15.el10.x86_64\    \ from CentOS-AppStream requires gcc = 14.0.1-0.15.el10, but none of the providers\    \ can be installed\n  - problem with installed package gcc-c++-11.4.1-2.1.el9.x86_64\n\    \  - package gcc-c++-11.4.1-2.1.el9.x86_64 from appstream requires libstdc++ =\    \ 11.4.1-2.1.el9, but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.1.el9.x86_64\    \ from @System requires libstdc++ = 11.4.1-2.1.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.3.1-4.4.el9.x86_64 from appstream requires\    \ libstdc++ = 11.3.1-4.4.el9, but none of the providers can be installed\n  -\    \ package gcc-c++-11.4.1-2.3.el9.x86_64 from appstream requires libstdc++ = 11.4.1-2.3.el9,\    \ but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.el9.x86_64\    \ from appstream requires libstdc++ = 11.4.1-2.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.4.1-3.el9.x86_64 from appstream requires\    \ libstdc++ = 11.4.1-3.el9, but none of the providers can be installed\n  - libstdc++-11.4.1-2.1.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.3.1-4.4.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - cannot install\    \ both libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS and libstdc++-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both libstdc++-11.3.1-4.4.el9.x86_64 from baseos\    \ and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot install\    \ both libstdc++-11.4.1-2.1.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-2.3.el9.x86_64\    \ from baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot\    \ install both libstdc++-11.4.1-2.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-3.el9.x86_64 from\    \ baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - package\    \ python3-hawkey-0.73.0-1.el10.x86_64 from CentOS-BaseOS requires libstdc++.so.6(GLIBCXX_3.4.30)(64bit),\    \ but none of the providers can be installed\n  - package python3-dnf-4.19.0-2.el10.noarch\    \ from CentOS-BaseOS requires python3-hawkey >= 0.73.0, but none of the providers\    \ can be installed\n  - package yum-utils-4.5.0-1.el10.noarch from CentOS-BaseOS\    \ requires python3-dnf >= 4.19.0, but none of the providers can be installed\n\    \  - cannot install the best candidate for the job"  rc: 1  results: []smithi026.front.sepia.ceph.com:  _ansible_no_log: null  changed: false  failures: []  invocation:    module_args:      allow_downgrade: false      allowerasing: false      autoremove: false      bugfix: false      cacheonly: false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin: []      disablerepo: []      download_dir: null      download_only: false      enable_plugin: []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps: true      installroot: /      list: null      lock_timeout: 30      name:      - dnf-utils      - sysstat      - libedit      - boost-thread      - xfsprogs      - gdisk      - parted      - libgcrypt      - fuse-libs      - openssl      - libuuid      - podman      - attr      - lsof      - gettext      - bc      - xfsdump      - blktrace      - usbredir      - valgrind      - nfs-utils      - ncurses-devel      - gcc      - git      - genisoimage      - qemu-img      - qemu-kvm-core      - qemu-kvm-block-rbd      - libacl-devel      - autoconf      - gdb      - iozone      nobest: false      releasever: null      security: false      skip_broken: false      sslverify: true      state: present      update_cache: false      update_only: false      validate_certs: true  msg: "Depsolve Error occurred: \n Problem: problem with installed package annobin-12.12-1.el9.x86_64\n\    \  - package annobin-12.12-1.el9.x86_64 from @System requires (gcc >= 11 with\    \ gcc < 12), but none of the providers can be installed\n  - package annobin-12.09-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.12-1.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - package\    \ annobin-12.28-1.el9.x86_64 from appstream requires (gcc >= 11 with gcc < 12),\    \ but none of the providers can be installed\n  - package annobin-12.31-1.el9.x86_64\    \ from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \ can be installed\n  - package annobin-12.31-2.el9.x86_64 from appstream requires\    \ (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - cannot\    \ install both gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream and gcc-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both gcc-11.3.1-4.4.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.1.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-2.3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \ gcc-11.4.1-2.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \ CentOS-AppStream\n  - cannot install both gcc-11.4.1-3.el9.x86_64 from appstream\    \ and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - package gcc-c++-14.0.1-0.15.el10.x86_64\    \ from CentOS-AppStream requires gcc = 14.0.1-0.15.el10, but none of the providers\    \ can be installed\n  - problem with installed package gcc-c++-11.4.1-2.1.el9.x86_64\n\    \  - package gcc-c++-11.4.1-2.1.el9.x86_64 from appstream requires libstdc++ =\    \ 11.4.1-2.1.el9, but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.1.el9.x86_64\    \ from @System requires libstdc++ = 11.4.1-2.1.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.3.1-4.4.el9.x86_64 from appstream requires\    \ libstdc++ = 11.3.1-4.4.el9, but none of the providers can be installed\n  -\    \ package gcc-c++-11.4.1-2.3.el9.x86_64 from appstream requires libstdc++ = 11.4.1-2.3.el9,\    \ but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.el9.x86_64\    \ from appstream requires libstdc++ = 11.4.1-2.el9, but none of the providers\    \ can be installed\n  - package gcc-c++-11.4.1-3.el9.x86_64 from appstream requires\    \ libstdc++ = 11.4.1-3.el9, but none of the providers can be installed\n  - libstdc++-11.4.1-2.1.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.3.1-4.4.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-3.el9.i686\    \ from baseos  does not belong to a distupgrade repository\n  - cannot install\    \ both libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS and libstdc++-11.4.1-2.1.el9.x86_64\    \ from @System\n  - cannot install both libstdc++-11.3.1-4.4.el9.x86_64 from baseos\    \ and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot install\    \ both libstdc++-11.4.1-2.1.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-2.3.el9.x86_64\    \ from baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot\    \ install both libstdc++-11.4.1-2.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \ from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-3.el9.x86_64 from\    \ baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - package\    \ python3-hawkey-0.73.0-1.el10.x86_64 from CentOS-BaseOS requires libstdc++.so.6(GLIBCXX_3.4.30)(64bit),\    \ but none of the providers can be installed\n  - package python3-dnf-4.19.0-2.el10.noarch\    \ from CentOS-BaseOS requires python3-hawkey >= 0.73.0, but none of the providers\    \ can be installed\n  - package yum-utils-4.5.0-1.el10.noarch from CentOS-BaseOS\    \ requires python3-dnf >= 4.19.0, but none of the providers can be installed\n\    \  - cannot install the best candidate for the job"  rc: 1  results: []
2024-04-19T02:38:52.053 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab
2024-04-19T02:38:52.064 INFO:teuthology.task.ansible:Skipping ansible cleanup...
2024-04-19T02:38:52.064 DEBUG:teuthology.run_tasks:Unwinding manager selinux
2024-04-19T02:38:52.071 DEBUG:teuthology.orchestra.run.smithi026:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log
2024-04-19T02:38:52.143 DEBUG:teuthology.orchestra.run.smithi164:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log
2024-04-19T02:38:52.211 DEBUG:teuthology.orchestra.run.smithi026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0
2024-04-19T02:38:52.246 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T02:38:52.246 DEBUG:teuthology.orchestra.run.smithi164:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0
2024-04-19T02:38:52.280 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T02:38:52.280 DEBUG:teuthology.run_tasks:Unwinding manager pcp
2024-04-19T02:38:52.288 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer
2024-04-19T02:38:52.296 INFO:teuthology.task.internal:Duration was 226.035326 seconds
2024-04-19T02:38:52.296 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog
2024-04-19T02:38:52.303 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring...
2024-04-19T02:38:52.303 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart
2024-04-19T02:38:52.307 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart
2024-04-19T02:38:52.355 INFO:teuthology.orchestra.run.smithi026.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-19T02:38:52.370 INFO:teuthology.orchestra.run.smithi164.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-19T02:38:52.663 INFO:teuthology.task.internal.syslog:Checking logs for errors...
2024-04-19T02:38:52.663 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi026.front.sepia.ceph.com
2024-04-19T02:38:52.663 DEBUG:teuthology.orchestra.run.smithi026:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1
2024-04-19T02:38:52.697 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi164.front.sepia.ceph.com
2024-04-19T02:38:52.697 DEBUG:teuthology.orchestra.run.smithi164:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1
2024-04-19T02:38:52.725 INFO:teuthology.task.internal.syslog:Compressing syslogs...
2024-04-19T02:38:52.725 DEBUG:teuthology.orchestra.run.smithi026:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
2024-04-19T02:38:52.740 DEBUG:teuthology.orchestra.run.smithi164:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
2024-04-19T02:38:52.799 INFO:teuthology.task.internal.syslog:Gathering journactl -b0...
2024-04-19T02:38:52.800 DEBUG:teuthology.orchestra.run.smithi026:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz
2024-04-19T02:38:52.820 DEBUG:teuthology.orchestra.run.smithi164:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz
2024-04-19T02:38:52.928 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo
2024-04-19T02:38:52.937 INFO:teuthology.task.internal:Restoring /etc/sudoers...
2024-04-19T02:38:52.937 DEBUG:teuthology.orchestra.run.smithi026:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers
2024-04-19T02:38:52.967 DEBUG:teuthology.orchestra.run.smithi164:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers
2024-04-19T02:38:52.997 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
2024-04-19T02:38:53.006 DEBUG:teuthology.orchestra.run.smithi026:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
2024-04-19T02:38:53.009 DEBUG:teuthology.orchestra.run.smithi164:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
2024-04-19T02:38:53.035 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern = core
2024-04-19T02:38:53.064 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern = core
2024-04-19T02:38:53.095 DEBUG:teuthology.orchestra.run.smithi026:> test -e /home/ubuntu/cephtest/archive/coredump
2024-04-19T02:38:53.113 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T02:38:53.113 DEBUG:teuthology.orchestra.run.smithi164:> test -e /home/ubuntu/cephtest/archive/coredump
2024-04-19T02:38:53.151 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-19T02:38:53.151 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive
2024-04-19T02:38:53.160 INFO:teuthology.task.internal:Transferring archived files...
2024-04-19T02:38:53.162 DEBUG:teuthology.misc:Transferring archived files from smithi026:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-19_02:08:09-upgrade-squid-distro-default-smithi/7663374/remote/smithi026
2024-04-19T02:38:53.163 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- .
2024-04-19T02:38:53.220 DEBUG:teuthology.misc:Transferring archived files from smithi164:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-19_02:08:09-upgrade-squid-distro-default-smithi/7663374/remote/smithi164
2024-04-19T02:38:53.221 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- .
2024-04-19T02:38:53.271 INFO:teuthology.task.internal:Removing archive directory...
2024-04-19T02:38:53.271 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf -- /home/ubuntu/cephtest/archive
2024-04-19T02:38:53.274 DEBUG:teuthology.orchestra.run.smithi164:> rm -rf -- /home/ubuntu/cephtest/archive
2024-04-19T02:38:53.322 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload
2024-04-19T02:38:53.333 INFO:teuthology.task.internal:Not uploading archives.
2024-04-19T02:38:53.333 DEBUG:teuthology.run_tasks:Unwinding manager internal.base
2024-04-19T02:38:53.366 INFO:teuthology.task.internal:Tidying up after the test...
2024-04-19T02:38:53.366 DEBUG:teuthology.orchestra.run.smithi026:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest
2024-04-19T02:38:53.368 DEBUG:teuthology.orchestra.run.smithi164:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest
2024-04-19T02:38:53.382 INFO:teuthology.orchestra.run.smithi026.stdout:   262166      4 drwxr-xr-x   2 ubuntu   ubuntu       4096 Apr 19 02:38 /home/ubuntu/cephtest
2024-04-19T02:38:53.384 INFO:teuthology.orchestra.run.smithi164.stdout:   262166      4 drwxr-xr-x   2 ubuntu   ubuntu       4096 Apr 19 02:38 /home/ubuntu/cephtest
2024-04-19T02:38:53.385 DEBUG:teuthology.run_tasks:Unwinding manager kernel
2024-04-19T02:38:53.395 DEBUG:teuthology.run_tasks:Unwinding manager console_log
2024-04-19T02:38:53.455 INFO:teuthology.nuke:Checking targets against current locks
2024-04-19T02:38:53.481 DEBUG:teuthology.nuke:shortname: smithi026
2024-04-19T02:38:53.482 INFO:teuthology.task.internal.check_lock:Checking locks...
2024-04-19T02:38:53.507 DEBUG:teuthology.nuke:shortname: smithi164
2024-04-19T02:38:53.507 INFO:teuthology.task.internal.check_lock:Checking locks...
2024-04-19T02:38:53.511 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi026.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-19_02:08:09-upgrade-squid-distro-default-smithi/7663374', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-19 02:25:44.042201', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFHKh6LkOrEvKFeTUma46CbsKIM0mlxXwveh6BrAQFNUGDo1UmKBqfZTTox3OECWzVshfAzYjdtdDavVYWfScO8='}
2024-04-19T02:38:53.536 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi164.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-19_02:08:09-upgrade-squid-distro-default-smithi/7663374', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-19 02:25:44.043565', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKnEg9Jg0W2+amaELbgkqJhxDUvGvyyW05vPjkzuWQi/XDd/P1V6iUoEq/lnFmC3ACliwmOaeAJvVJy48sxKi2Q='}
2024-04-19T02:38:53.572 INFO:teuthology.orchestra.console.smithi026:Power off
2024-04-19T02:38:53.572 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2024-04-19T02:38:53.598 DEBUG:teuthology.orchestra.console.smithi026:power off output: Chassis Power Control: Down/Off
2024-04-19T02:38:53.598 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T02:38:53.615 INFO:teuthology.orchestra.console.smithi164:Power off
2024-04-19T02:38:53.615 DEBUG:teuthology.orchestra.console.smithi164:pexpect command: ipmitool -H smithi164.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2024-04-19T02:38:53.627 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is on
2024-04-19T02:38:53.641 DEBUG:teuthology.orchestra.console.smithi164:power off output: Chassis Power Control: Down/Off
2024-04-19T02:38:53.641 DEBUG:teuthology.orchestra.console.smithi164:pexpect command: ipmitool -H smithi164.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T02:38:53.666 DEBUG:teuthology.orchestra.console.smithi164:check power output: Chassis Power is on
2024-04-19T02:38:57.629 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T02:38:57.667 DEBUG:teuthology.orchestra.console.smithi164:pexpect command: ipmitool -H smithi164.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T02:38:57.741 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is on
2024-04-19T02:38:57.778 DEBUG:teuthology.orchestra.console.smithi164:check power output: Chassis Power is on
2024-04-19T02:39:01.746 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T02:39:01.779 DEBUG:teuthology.orchestra.console.smithi164:pexpect command: ipmitool -H smithi164.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T02:39:01.857 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is off
2024-04-19T02:39:01.889 DEBUG:teuthology.orchestra.console.smithi164:check power output: Chassis Power is on
2024-04-19T02:39:01.958 INFO:teuthology.orchestra.console.smithi026:Power off completed
2024-04-19T02:39:05.895 DEBUG:teuthology.orchestra.console.smithi164:pexpect command: ipmitool -H smithi164.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-19T02:39:06.006 DEBUG:teuthology.orchestra.console.smithi164:check power output: Chassis Power is off
2024-04-19T02:39:06.107 INFO:teuthology.orchestra.console.smithi164:Power off completed
2024-04-19T02:39:06.242 INFO:teuthology.run:Summary data:
description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start
  2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export
  rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
duration: 226.03532552719116
failure_reason: 'smithi164.front.sepia.ceph.com:  _ansible_no_log: null  changed:
  false  failures: []  invocation:    module_args:      allow_downgrade: false      allowerasing:
  false      autoremove: false      bugfix: false      cacheonly: false      conf_file:
  null      disable_excludes: null      disable_gpg_check: false      disable_plugin:
  []      disablerepo: []      download_dir: null      download_only: false      enable_plugin:
  []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps:
  true      installroot: /      list: null      lock_timeout: 30      name:      -
  dnf-utils      - sysstat      - libedit      - boost-thread      - xfsprogs      -
  gdisk      - parted      - libgcrypt      - fuse-libs      - openssl      - libuuid      -
  podman      - attr      - lsof      - gettext      - bc      - xfsdump      - blktrace      -
  usbredir      - valgrind      - nfs-utils      - ncurses-devel      - gcc      -
  git      - genisoimage      - qemu-img      - qemu-kvm-core      - qemu-kvm-block-rbd      -
  libacl-devel      - autoconf      - gdb      - iozone      nobest: false      releasever:
  null      security: false      skip_broken: false      sslverify: true      state:
  present      update_cache: false      update_only: false      validate_certs: true  msg:
  "Depsolve Error occurred: \n Problem: problem with installed package annobin-12.12-1.el9.x86_64\n\    \  -
  package annobin-12.12-1.el9.x86_64 from @System requires (gcc >= 11 with\    \ gcc
  < 12), but none of the providers can be installed\n  - package annobin-12.09-1.el9.x86_64\    \
  from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \
  can be installed\n  - package annobin-12.12-1.el9.x86_64 from appstream requires\    \
  (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - package\    \
  annobin-12.28-1.el9.x86_64 from appstream requires (gcc >= 11 with gcc < 12),\    \
  but none of the providers can be installed\n  - package annobin-12.31-1.el9.x86_64\    \
  from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \
  can be installed\n  - package annobin-12.31-2.el9.x86_64 from appstream requires\    \
  (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - cannot\    \
  install both gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream and gcc-11.4.1-2.1.el9.x86_64\    \
  from @System\n  - cannot install both gcc-11.3.1-4.4.el9.x86_64 from appstream\    \
  and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \
  gcc-11.4.1-2.1.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \
  CentOS-AppStream\n  - cannot install both gcc-11.4.1-2.3.el9.x86_64 from appstream\    \
  and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \
  gcc-11.4.1-2.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \
  CentOS-AppStream\n  - cannot install both gcc-11.4.1-3.el9.x86_64 from appstream\    \
  and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - package gcc-c++-14.0.1-0.15.el10.x86_64\    \
  from CentOS-AppStream requires gcc = 14.0.1-0.15.el10, but none of the providers\    \
  can be installed\n  - problem with installed package gcc-c++-11.4.1-2.1.el9.x86_64\n\    \  -
  package gcc-c++-11.4.1-2.1.el9.x86_64 from appstream requires libstdc++ =\    \
  11.4.1-2.1.el9, but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.1.el9.x86_64\    \
  from @System requires libstdc++ = 11.4.1-2.1.el9, but none of the providers\    \
  can be installed\n  - package gcc-c++-11.3.1-4.4.el9.x86_64 from appstream requires\    \
  libstdc++ = 11.3.1-4.4.el9, but none of the providers can be installed\n  -\    \
  package gcc-c++-11.4.1-2.3.el9.x86_64 from appstream requires libstdc++ = 11.4.1-2.3.el9,\    \
  but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.el9.x86_64\    \
  from appstream requires libstdc++ = 11.4.1-2.el9, but none of the providers\    \
  can be installed\n  - package gcc-c++-11.4.1-3.el9.x86_64 from appstream requires\    \
  libstdc++ = 11.4.1-3.el9, but none of the providers can be installed\n  - libstdc++-11.4.1-2.1.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.3.1-4.4.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.3.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-3.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - cannot install\    \
  both libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS and libstdc++-11.4.1-2.1.el9.x86_64\    \
  from @System\n  - cannot install both libstdc++-11.3.1-4.4.el9.x86_64 from baseos\    \
  and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot install\    \
  both libstdc++-11.4.1-2.1.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \
  from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-2.3.el9.x86_64\    \
  from baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot\    \
  install both libstdc++-11.4.1-2.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \
  from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-3.el9.x86_64 from\    \
  baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - package\    \
  python3-hawkey-0.73.0-1.el10.x86_64 from CentOS-BaseOS requires libstdc++.so.6(GLIBCXX_3.4.30)(64bit),\    \
  but none of the providers can be installed\n  - package python3-dnf-4.19.0-2.el10.noarch\    \
  from CentOS-BaseOS requires python3-hawkey >= 0.73.0, but none of the providers\    \
  can be installed\n  - package yum-utils-4.5.0-1.el10.noarch from CentOS-BaseOS\    \
  requires python3-dnf >= 4.19.0, but none of the providers can be installed\n\    \  -
  cannot install the best candidate for the job"  rc: 1  results: []smithi026.front.sepia.ceph.com:  _ansible_no_log:
  null  changed: false  failures: []  invocation:    module_args:      allow_downgrade:
  false      allowerasing: false      autoremove: false      bugfix: false      cacheonly:
  false      conf_file: null      disable_excludes: null      disable_gpg_check: false      disable_plugin:
  []      disablerepo: []      download_dir: null      download_only: false      enable_plugin:
  []      enablerepo: []      exclude: []      install_repoquery: true      install_weak_deps:
  true      installroot: /      list: null      lock_timeout: 30      name:      -
  dnf-utils      - sysstat      - libedit      - boost-thread      - xfsprogs      -
  gdisk      - parted      - libgcrypt      - fuse-libs      - openssl      - libuuid      -
  podman      - attr      - lsof      - gettext      - bc      - xfsdump      - blktrace      -
  usbredir      - valgrind      - nfs-utils      - ncurses-devel      - gcc      -
  git      - genisoimage      - qemu-img      - qemu-kvm-core      - qemu-kvm-block-rbd      -
  libacl-devel      - autoconf      - gdb      - iozone      nobest: false      releasever:
  null      security: false      skip_broken: false      sslverify: true      state:
  present      update_cache: false      update_only: false      validate_certs: true  msg:
  "Depsolve Error occurred: \n Problem: problem with installed package annobin-12.12-1.el9.x86_64\n\    \  -
  package annobin-12.12-1.el9.x86_64 from @System requires (gcc >= 11 with\    \ gcc
  < 12), but none of the providers can be installed\n  - package annobin-12.09-1.el9.x86_64\    \
  from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \
  can be installed\n  - package annobin-12.12-1.el9.x86_64 from appstream requires\    \
  (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - package\    \
  annobin-12.28-1.el9.x86_64 from appstream requires (gcc >= 11 with gcc < 12),\    \
  but none of the providers can be installed\n  - package annobin-12.31-1.el9.x86_64\    \
  from appstream requires (gcc >= 11 with gcc < 12), but none of the providers\    \
  can be installed\n  - package annobin-12.31-2.el9.x86_64 from appstream requires\    \
  (gcc >= 11 with gcc < 12), but none of the providers can be installed\n  - cannot\    \
  install both gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream and gcc-11.4.1-2.1.el9.x86_64\    \
  from @System\n  - cannot install both gcc-11.3.1-4.4.el9.x86_64 from appstream\    \
  and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \
  gcc-11.4.1-2.1.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \
  CentOS-AppStream\n  - cannot install both gcc-11.4.1-2.3.el9.x86_64 from appstream\    \
  and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - cannot install both\    \
  gcc-11.4.1-2.el9.x86_64 from appstream and gcc-14.0.1-0.15.el10.x86_64 from\    \
  CentOS-AppStream\n  - cannot install both gcc-11.4.1-3.el9.x86_64 from appstream\    \
  and gcc-14.0.1-0.15.el10.x86_64 from CentOS-AppStream\n  - package gcc-c++-14.0.1-0.15.el10.x86_64\    \
  from CentOS-AppStream requires gcc = 14.0.1-0.15.el10, but none of the providers\    \
  can be installed\n  - problem with installed package gcc-c++-11.4.1-2.1.el9.x86_64\n\    \  -
  package gcc-c++-11.4.1-2.1.el9.x86_64 from appstream requires libstdc++ =\    \
  11.4.1-2.1.el9, but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.1.el9.x86_64\    \
  from @System requires libstdc++ = 11.4.1-2.1.el9, but none of the providers\    \
  can be installed\n  - package gcc-c++-11.3.1-4.4.el9.x86_64 from appstream requires\    \
  libstdc++ = 11.3.1-4.4.el9, but none of the providers can be installed\n  -\    \
  package gcc-c++-11.4.1-2.3.el9.x86_64 from appstream requires libstdc++ = 11.4.1-2.3.el9,\    \
  but none of the providers can be installed\n  - package gcc-c++-11.4.1-2.el9.x86_64\    \
  from appstream requires libstdc++ = 11.4.1-2.el9, but none of the providers\    \
  can be installed\n  - package gcc-c++-11.4.1-3.el9.x86_64 from appstream requires\    \
  libstdc++ = 11.4.1-3.el9, but none of the providers can be installed\n  - libstdc++-11.4.1-2.1.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.3.1-4.4.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.3.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-2.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - libstdc++-11.4.1-3.el9.i686\    \
  from baseos  does not belong to a distupgrade repository\n  - cannot install\    \
  both libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS and libstdc++-11.4.1-2.1.el9.x86_64\    \
  from @System\n  - cannot install both libstdc++-11.3.1-4.4.el9.x86_64 from baseos\    \
  and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot install\    \
  both libstdc++-11.4.1-2.1.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \
  from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-2.3.el9.x86_64\    \
  from baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - cannot\    \
  install both libstdc++-11.4.1-2.el9.x86_64 from baseos and libstdc++-14.0.1-0.15.el10.x86_64\    \
  from CentOS-BaseOS\n  - cannot install both libstdc++-11.4.1-3.el9.x86_64 from\    \
  baseos and libstdc++-14.0.1-0.15.el10.x86_64 from CentOS-BaseOS\n  - package\    \
  python3-hawkey-0.73.0-1.el10.x86_64 from CentOS-BaseOS requires libstdc++.so.6(GLIBCXX_3.4.30)(64bit),\    \
  but none of the providers can be installed\n  - package python3-dnf-4.19.0-2.el10.noarch\    \
  from CentOS-BaseOS requires python3-hawkey >= 0.73.0, but none of the providers\    \
  can be installed\n  - package yum-utils-4.5.0-1.el10.noarch from CentOS-BaseOS\    \
  requires python3-dnf >= 4.19.0, but none of the providers can be installed\n\    \  -
  cannot install the best candidate for the job"  rc: 1  results: []'
owner: scheduled_teuthology@teuthology
sentry_event: https://sentry.ceph.com/organizations/ceph/?query=c26ad245b4884e8990a5065af425dbc8
status: dead
success: false

2024-04-19T02:39:06.242 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/
2024-04-19T02:39:06.321 INFO:teuthology.run:DEAD