2024-04-10T22:37:23.923 INFO:root:teuthology version: 0.0.1.dev267+g6c63784
2024-04-10T22:37:23.924 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} --verbose --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650186 --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650186/orig.config.yaml
2024-04-10T22:37:23.985 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/
2024-04-10T22:37:24.068 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650186
  branch: reef
  description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8
    0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls
    rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench
    mon_election/classic}
  email: ceph-qa@ceph.com
  first-half-sequence:
  - cephadm.shell:
      env:
      - sha1
      mon.a:
      - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60
      - ceph config set global log_to_journald false --force
      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
      - ceph orch ps
      - echo wait for minority of mons to upgrade
      - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done
      - ceph orch ps
      - ceph orch upgrade pause
      - sleep 60
      - ceph orch upgrade resume
      - echo wait for majority of mons to upgrade
      - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done'
      - ceph orch ps
      - ceph orch upgrade pause
      - sleep 60
      - ceph orch upgrade resume
      - echo wait for all mons to upgrade
      - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done'
      - ceph orch ps
      - ceph orch upgrade pause
      - sleep 60
      - ceph orch upgrade resume
      - echo wait for half of osds to upgrade
      - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ;
        done'
      - ceph orch upgrade pause
      - ceph orch ps
      - ceph orch ps
      - ceph versions
  first-half-tasks:
  - full_sequential:
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
  - print: '**** done end radosbench.yaml'
  first_in_suite: false
  job_id: '7650186'
  kernel:
    kdb: 1
    sha1: distro
  last_in_suite: false
  machine_type: smithi
  meta:
  - desc: 'run randomized correctness test for rados operations
  
      generate write load with rados bench
  
      '
  - desc: 'run randomized correctness test for rados operations
  
      generate write load with rados bench
  
      '
  - desc: 'run basic cls tests for rbd
  
      '
  - desc: 'run basic import/export cli tests for rbd
  
      '
  - desc: 'librbd C and C++ api tests
  
      '
  - desc: 'randomized correctness test for rados operations on a replicated pool,
  
      using only reads, writes, and deletes
  
      '
  - desc: 'randomized correctness test for rados operations on a replicated pool with
      snapshot operations
  
      '
  - desc: 'run randomized correctness test for rados operations
  
      generate write load with rados bench
  
      '
  name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi
  no_nested_subset: false
  nuke-on-error: true
  openstack:
  - volumes:
      count: 4
      size: 10
  os_type: rhel
  os_version: '8.6'
  overrides:
    admin_socket:
      branch: reef
    ceph:
      conf:
        global:
          mon election default strategy: 1
        mgr:
          debug mgr: 20
          debug ms: 1
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
        osd:
          debug ms: 1
          debug osd: 20
          osd shutdown pgref assert: true
      create_rbd_pool: true
      flavor: default
      log-ignorelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42
    ceph-deploy:
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon: {}
    install:
      ceph:
        flavor: default
        sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42
    selinux:
      whitelist:
      - scontext=system_u:system_r:logrotate_t:s0
    workunit:
      branch: reef
      sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c
  owner: scheduled_teuthology@teuthology
  priority: 830
  repo: https://git.ceph.com/ceph.git
  roles:
  - - mon.a
    - mon.c
    - mgr.y
    - osd.0
    - osd.1
    - osd.2
    - osd.3
    - client.0
    - node-exporter.a
    - alertmanager.a
  - - mon.b
    - mgr.x
    - osd.4
    - osd.5
    - osd.6
    - osd.7
    - client.1
    - prometheus.a
    - grafana.a
    - node-exporter.b
  second-half-sequence:
    sequential:
    - cephadm.shell:
        env:
        - sha1
        mon.a:
        - ceph orch upgrade resume
        - sleep 60
        - echo wait for upgrade to complete
        - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch
          ps ; ceph versions ; sleep 30 ; done
        - echo upgrade complete
        - ceph orch ps
        - ceph versions
        - ceph versions | jq -e '.overall | length == 1'
        - ceph versions | jq -e '.overall | keys' | grep $sha1
  second-half-tasks:
  - full_sequential:
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
  - print: '**** done end radosbench.yaml'
  seed: 4604
  sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42
  sleep_before_teardown: 0
  stress-tasks:
  - thrashosds:
      aggressive_pg_num_changes: false
      chance_force_recovery: 0
      chance_pgnum_grow: 1
      chance_pgpnum_fix: 1
      chance_thrash_cluster_full: 0
      chance_thrash_pg_upmap: 0
      chance_thrash_pg_upmap_items: 0
      disable_objectstore_tool_tests: true
      timeout: 1200
  - full_sequential:
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
  - print: '**** done end radosbench.yaml'
  - workunit:
      branch: pacific
      clients:
        client.0:
        - cls/test_cls_rbd.sh
  - print: '**** done cls/test_cls_rbd.sh 5-workload'
  - workunit:
      branch: pacific
      clients:
        client.0:
        - rbd/import_export.sh
      env:
        RBD_CREATE_ARGS: --new-format
  - print: '**** done rbd/import_export.sh 5-workload'
  - workunit:
      branch: pacific
      clients:
        client.0:
        - rbd/test_librbd.sh
      env:
        RBD_FEATURES: '61'
  - print: '**** done rbd/test_librbd.sh 7-workload'
  - full_sequential:
    - rados:
        clients:
        - client.0
        objects: 500
        op_weights:
          delete: 10
          read: 45
          write: 45
        ops: 4000
        write_append_excl: false
  - print: '**** done rados/readwrite 5-workload'
  - full_sequential:
    - rados:
        clients:
        - client.0
        objects: 50
        op_weights:
          delete: 50
          read: 100
          rollback: 50
          snap_create: 50
          snap_remove: 50
          write: 100
        ops: 4000
        write_append_excl: false
  - print: '**** done rados/snaps-few-objects 5-workload'
  subset: 0/1
  suite: upgrade:pacific-x
  suite_branch: reef
  suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa
  suite_relpath: qa
  suite_repo: https://git.ceph.com/ceph.git
  suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c
  targets:
    smithi118.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP9+bjT+d23cXpNHyWMczgMDvi2F+N89PzP8dIT+PvPP+mZ7zjds37K3I3l1T1rSKFATCEvqTOShWlr8GC4qDSY=
    smithi176.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLloNgZVPx4NOZ6z6LwGKk2JoLLrAgyLHbfDjHK88gkCj5k10tOApVyo2RVha6z1X3iqkcVJrMWZl3ZuiJxoHNo=
  tasks:
  - pexec:
      all:
      - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
      - sudo dnf -y  module reset container-tools
      - sudo dnf -y  module install container-tools:rhel8 --allowerasing --nobest
      - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
  - install:
      branch: pacific
      exclude_packages:
      - ceph-volume
  - cephadm:
      cephadm_branch: pacific
      cephadm_git_url: https://github.com/ceph/ceph
      conf:
        osd:
          osd_class_default_list: '*'
          osd_class_load_list: '*'
      image: quay.ceph.io/ceph-ci/ceph:pacific
  - cephadm.shell:
      mon.a:
      - ceph fs volume create foo
      - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
      - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
  - ceph.healthy: null
  - print: '**** upgrading first half of cluster, with stress ****'
  - parallel:
    - first-half-tasks
    - first-half-sequence
  - print: '**** done upgrading first half of cluster ****'
  - ceph.healthy: null
  - print: '**** applying stress + thrashing to mixed-version cluster ****'
  - parallel:
    - stress-tasks
  - ceph.healthy: null
  - print: '**** finishing upgrade ****'
  - parallel:
    - second-half-tasks
    - second-half-sequence
  - ceph.healthy: null
  teuthology:
    fragments_dropped: []
    meta: {}
    postmerge: []
  teuthology_branch: main
  teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab
  timestamp: 2024-04-10_01:08:08
  tube: smithi
  user: teuthology
  verbose: false
  worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885
2024-04-10T22:37:24.069 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it
2024-04-10T22:37:24.069 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks
2024-04-10T22:37:24.069 INFO:teuthology.run_tasks:Running task internal.check_packages...
2024-04-10T22:37:24.071 INFO:teuthology.task.internal:Checking packages...
2024-04-10T22:37:24.093 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42'
2024-04-10T22:37:24.093 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2024-04-10T22:37:24.093 INFO:teuthology.packaging:ref: None
2024-04-10T22:37:24.093 INFO:teuthology.packaging:tag: None
2024-04-10T22:37:24.093 INFO:teuthology.packaging:branch: reef
2024-04-10T22:37:24.094 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42
2024-04-10T22:37:24.094 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef
2024-04-10T22:37:24.460 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47
2024-04-10T22:37:24.461 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2024-04-10T22:37:24.468 INFO:teuthology.task.internal:no buildpackages task found
2024-04-10T22:37:24.468 INFO:teuthology.run_tasks:Running task internal.save_config...
2024-04-10T22:37:24.474 INFO:teuthology.task.internal:Saving configuration
2024-04-10T22:37:24.497 INFO:teuthology.run_tasks:Running task internal.check_lock...
2024-04-10T22:37:24.503 INFO:teuthology.task.internal.check_lock:Checking locks...
2024-04-10T22:37:24.530 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi118.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650186', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:17.965214', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP9+bjT+d23cXpNHyWMczgMDvi2F+N89PzP8dIT+PvPP+mZ7zjds37K3I3l1T1rSKFATCEvqTOShWlr8GC4qDSY='}
2024-04-10T22:37:24.555 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi176.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650186', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:17.963494', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLloNgZVPx4NOZ6z6LwGKk2JoLLrAgyLHbfDjHK88gkCj5k10tOApVyo2RVha6z1X3iqkcVJrMWZl3ZuiJxoHNo='}
2024-04-10T22:37:24.556 INFO:teuthology.run_tasks:Running task internal.add_remotes...
2024-04-10T22:37:24.562 INFO:teuthology.task.internal:roles: ubuntu@smithi118.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a']
2024-04-10T22:37:24.562 INFO:teuthology.task.internal:roles: ubuntu@smithi176.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b']
2024-04-10T22:37:24.562 INFO:teuthology.run_tasks:Running task console_log...
2024-04-10T22:37:24.637 DEBUG:teuthology.exit:Installing handler: Handler(exiter=<teuthology.exit.Exiter object at 0x7fc33f41a790>, func=<function ConsoleLog.start_logging.<locals>.kill_console_loggers at 0x7fc3402060d0>, signals=[15])
2024-04-10T22:37:24.638 INFO:teuthology.run_tasks:Running task internal.connect...
2024-04-10T22:37:24.644 INFO:teuthology.task.internal:Opening connections...
2024-04-10T22:37:24.645 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi118.front.sepia.ceph.com
2024-04-10T22:37:24.646 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi118.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-10T22:37:24.719 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi176.front.sepia.ceph.com
2024-04-10T22:37:24.720 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi176.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-10T22:37:24.789 INFO:teuthology.run_tasks:Running task internal.push_inventory...
2024-04-10T22:37:24.802 DEBUG:teuthology.orchestra.run.smithi118:> uname -m
2024-04-10T22:37:24.823 INFO:teuthology.orchestra.run.smithi118.stdout:x86_64
2024-04-10T22:37:24.823 DEBUG:teuthology.orchestra.run.smithi118:> cat /etc/os-release
2024-04-10T22:37:24.883 INFO:teuthology.orchestra.run.smithi118.stdout:NAME="Red Hat Enterprise Linux"
2024-04-10T22:37:24.883 INFO:teuthology.orchestra.run.smithi118.stdout:VERSION="8.6 (Ootpa)"
2024-04-10T22:37:24.883 INFO:teuthology.orchestra.run.smithi118.stdout:ID="rhel"
2024-04-10T22:37:24.883 INFO:teuthology.orchestra.run.smithi118.stdout:ID_LIKE="fedora"
2024-04-10T22:37:24.883 INFO:teuthology.orchestra.run.smithi118.stdout:VERSION_ID="8.6"
2024-04-10T22:37:24.883 INFO:teuthology.orchestra.run.smithi118.stdout:PLATFORM_ID="platform:el8"
2024-04-10T22:37:24.883 INFO:teuthology.orchestra.run.smithi118.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)"
2024-04-10T22:37:24.884 INFO:teuthology.orchestra.run.smithi118.stdout:ANSI_COLOR="0;31"
2024-04-10T22:37:24.884 INFO:teuthology.orchestra.run.smithi118.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos"
2024-04-10T22:37:24.884 INFO:teuthology.orchestra.run.smithi118.stdout:HOME_URL="https://www.redhat.com/"
2024-04-10T22:37:24.884 INFO:teuthology.orchestra.run.smithi118.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/"
2024-04-10T22:37:24.884 INFO:teuthology.orchestra.run.smithi118.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/"
2024-04-10T22:37:24.884 INFO:teuthology.orchestra.run.smithi118.stdout:
2024-04-10T22:37:24.884 INFO:teuthology.orchestra.run.smithi118.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8"
2024-04-10T22:37:24.884 INFO:teuthology.orchestra.run.smithi118.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6
2024-04-10T22:37:24.884 INFO:teuthology.orchestra.run.smithi118.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
2024-04-10T22:37:24.884 INFO:teuthology.orchestra.run.smithi118.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6"
2024-04-10T22:37:24.885 INFO:teuthology.lock.ops:Updating smithi118.front.sepia.ceph.com on lock server
2024-04-10T22:37:24.903 DEBUG:teuthology.orchestra.run.smithi176:> uname -m
2024-04-10T22:37:24.924 INFO:teuthology.orchestra.run.smithi176.stdout:x86_64
2024-04-10T22:37:24.925 DEBUG:teuthology.orchestra.run.smithi176:> cat /etc/os-release
2024-04-10T22:37:24.984 INFO:teuthology.orchestra.run.smithi176.stdout:NAME="Red Hat Enterprise Linux"
2024-04-10T22:37:24.984 INFO:teuthology.orchestra.run.smithi176.stdout:VERSION="8.6 (Ootpa)"
2024-04-10T22:37:24.984 INFO:teuthology.orchestra.run.smithi176.stdout:ID="rhel"
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:ID_LIKE="fedora"
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:VERSION_ID="8.6"
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:PLATFORM_ID="platform:el8"
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)"
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:ANSI_COLOR="0;31"
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos"
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:HOME_URL="https://www.redhat.com/"
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/"
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/"
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8"
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
2024-04-10T22:37:24.985 INFO:teuthology.orchestra.run.smithi176.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6"
2024-04-10T22:37:24.986 INFO:teuthology.lock.ops:Updating smithi176.front.sepia.ceph.com on lock server
2024-04-10T22:37:25.002 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles...
2024-04-10T22:37:25.012 INFO:teuthology.run_tasks:Running task internal.check_conflict...
2024-04-10T22:37:25.018 INFO:teuthology.task.internal:Checking for old test directory...
2024-04-10T22:37:25.018 DEBUG:teuthology.orchestra.run.smithi118:> test '!' -e /home/ubuntu/cephtest
2024-04-10T22:37:25.021 DEBUG:teuthology.orchestra.run.smithi176:> test '!' -e /home/ubuntu/cephtest
2024-04-10T22:37:25.043 INFO:teuthology.run_tasks:Running task internal.check_ceph_data...
2024-04-10T22:37:25.054 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph...
2024-04-10T22:37:25.054 DEBUG:teuthology.orchestra.run.smithi118:> test -z $(ls -A /var/lib/ceph)
2024-04-10T22:37:25.079 DEBUG:teuthology.orchestra.run.smithi176:> test -z $(ls -A /var/lib/ceph)
2024-04-10T22:37:25.117 INFO:teuthology.run_tasks:Running task internal.vm_setup...
2024-04-10T22:37:25.172 INFO:teuthology.run_tasks:Running task kernel...
2024-04-10T22:37:25.186 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'}
2024-04-10T22:37:25.186 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300
2024-04-10T22:37:25.186 DEBUG:teuthology.orchestra.run.smithi118:> test -f /run/.containerenv -o -f /.dockerenv
2024-04-10T22:37:25.187 DEBUG:teuthology.orchestra.run.smithi176:> test -f /run/.containerenv -o -f /.dockerenv
2024-04-10T22:37:25.205 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:37:25.206 DEBUG:teuthology.orchestra.run.smithi118:> uname -r
2024-04-10T22:37:25.207 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:37:25.207 DEBUG:teuthology.orchestra.run.smithi176:> uname -r
2024-04-10T22:37:25.267 INFO:teuthology.orchestra.run.smithi176.stdout:4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:37:25.267 INFO:teuthology.orchestra.run.smithi118.stdout:4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:37:25.267 INFO:teuthology.task.kernel:Running kernel on smithi176: 4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:37:25.267 DEBUG:teuthology.orchestra.run.smithi176:> sudo yum install -y kernel
2024-04-10T22:37:25.267 INFO:teuthology.task.kernel:Running kernel on smithi118: 4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:37:25.267 DEBUG:teuthology.orchestra.run.smithi118:> sudo yum install -y kernel
2024-04-10T22:37:26.798 INFO:teuthology.orchestra.run.smithi118.stdout:Updating Subscription Management repositories.
2024-04-10T22:37:26.798 INFO:teuthology.orchestra.run.smithi118.stdout:Unable to read consumer identity
2024-04-10T22:37:26.826 INFO:teuthology.orchestra.run.smithi176.stdout:Updating Subscription Management repositories.
2024-04-10T22:37:26.826 INFO:teuthology.orchestra.run.smithi176.stdout:Unable to read consumer identity
2024-04-10T22:37:26.899 INFO:teuthology.orchestra.run.smithi118.stdout:
2024-04-10T22:37:26.899 INFO:teuthology.orchestra.run.smithi118.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register.
2024-04-10T22:37:26.899 INFO:teuthology.orchestra.run.smithi118.stdout:
2024-04-10T22:37:26.930 INFO:teuthology.orchestra.run.smithi176.stdout:
2024-04-10T22:37:26.930 INFO:teuthology.orchestra.run.smithi176.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register.
2024-04-10T22:37:26.931 INFO:teuthology.orchestra.run.smithi176.stdout:
2024-04-10T22:38:14.891 INFO:teuthology.orchestra.run.smithi176.stdout:Copr repo for python3-asyncssh owned by ceph    0.0  B/s |   0  B     00:46
2024-04-10T22:38:14.892 INFO:teuthology.orchestra.run.smithi176.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh':
2024-04-10T22:38:14.892 INFO:teuthology.orchestra.run.smithi176.stderr:  - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org]
2024-04-10T22:38:14.892 INFO:teuthology.orchestra.run.smithi176.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried
2024-04-10T22:38:16.075 INFO:teuthology.orchestra.run.smithi176.stdout:Extra Packages for Enterprise Linux              14 MB/s |  16 MB     00:01
2024-04-10T22:38:19.623 INFO:teuthology.orchestra.run.smithi176.stdout:lab-extras                                      382 kB/s |  24 kB     00:00
2024-04-10T22:38:19.697 INFO:teuthology.orchestra.run.smithi176.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh
2024-04-10T22:38:20.342 INFO:teuthology.orchestra.run.smithi176.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed.
2024-04-10T22:38:20.343 INFO:teuthology.orchestra.run.smithi176.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed.
2024-04-10T22:38:20.422 INFO:teuthology.orchestra.run.smithi176.stdout:Dependencies resolved.
2024-04-10T22:38:20.423 INFO:teuthology.orchestra.run.smithi176.stdout:Nothing to do.
2024-04-10T22:38:20.424 INFO:teuthology.orchestra.run.smithi176.stdout:Complete!
2024-04-10T22:38:20.572 DEBUG:teuthology.orchestra.run.smithi176:> echo no | sudo yum reinstall kernel || true
2024-04-10T22:38:20.913 INFO:teuthology.orchestra.run.smithi176.stdout:Updating Subscription Management repositories.
2024-04-10T22:38:20.914 INFO:teuthology.orchestra.run.smithi176.stdout:Unable to read consumer identity
2024-04-10T22:38:20.967 INFO:teuthology.orchestra.run.smithi176.stdout:
2024-04-10T22:38:20.968 INFO:teuthology.orchestra.run.smithi176.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register.
2024-04-10T22:38:20.968 INFO:teuthology.orchestra.run.smithi176.stdout:
2024-04-10T22:38:23.280 INFO:teuthology.orchestra.run.smithi118.stdout:Copr repo for python3-asyncssh owned by ceph    0.0  B/s |   0  B     00:53
2024-04-10T22:38:23.280 INFO:teuthology.orchestra.run.smithi118.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh':
2024-04-10T22:38:23.280 INFO:teuthology.orchestra.run.smithi118.stderr:  - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org]
2024-04-10T22:38:23.280 INFO:teuthology.orchestra.run.smithi118.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried
2024-04-10T22:38:24.866 INFO:teuthology.orchestra.run.smithi118.stdout:Extra Packages for Enterprise Linux              10 MB/s |  16 MB     00:01
2024-04-10T22:38:28.405 INFO:teuthology.orchestra.run.smithi118.stdout:lab-extras                                      581 kB/s |  24 kB     00:00
2024-04-10T22:38:28.480 INFO:teuthology.orchestra.run.smithi118.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh
2024-04-10T22:38:29.074 INFO:teuthology.orchestra.run.smithi118.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed.
2024-04-10T22:38:29.075 INFO:teuthology.orchestra.run.smithi118.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed.
2024-04-10T22:38:29.189 INFO:teuthology.orchestra.run.smithi118.stdout:Dependencies resolved.
2024-04-10T22:38:29.190 INFO:teuthology.orchestra.run.smithi118.stdout:Nothing to do.
2024-04-10T22:38:29.190 INFO:teuthology.orchestra.run.smithi118.stdout:Complete!
2024-04-10T22:38:29.335 DEBUG:teuthology.orchestra.run.smithi118:> echo no | sudo yum reinstall kernel || true
2024-04-10T22:38:29.673 INFO:teuthology.orchestra.run.smithi118.stdout:Updating Subscription Management repositories.
2024-04-10T22:38:29.673 INFO:teuthology.orchestra.run.smithi118.stdout:Unable to read consumer identity
2024-04-10T22:38:29.733 INFO:teuthology.orchestra.run.smithi118.stdout:
2024-04-10T22:38:29.733 INFO:teuthology.orchestra.run.smithi118.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register.
2024-04-10T22:38:29.733 INFO:teuthology.orchestra.run.smithi118.stdout:
2024-04-10T22:39:14.260 INFO:teuthology.orchestra.run.smithi176.stdout:Copr repo for python3-asyncssh owned by ceph    0.0  B/s |   0  B     00:53
2024-04-10T22:39:14.261 INFO:teuthology.orchestra.run.smithi176.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh':
2024-04-10T22:39:14.261 INFO:teuthology.orchestra.run.smithi176.stderr:  - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org]
2024-04-10T22:39:14.261 INFO:teuthology.orchestra.run.smithi176.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried
2024-04-10T22:39:14.421 INFO:teuthology.orchestra.run.smithi176.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh
2024-04-10T22:39:14.422 INFO:teuthology.orchestra.run.smithi176.stdout:Last metadata expiration check: 0:00:55 ago on Wed 10 Apr 2024 10:38:19 PM UTC.
2024-04-10T22:39:14.500 INFO:teuthology.orchestra.run.smithi176.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2024-04-10T22:39:14.503 INFO:teuthology.orchestra.run.smithi176.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available.
2024-04-10T22:39:14.547 INFO:teuthology.orchestra.run.smithi176.stderr:Error: No packages marked for reinstall.
2024-04-10T22:39:14.595 DEBUG:teuthology.orchestra.run.smithi176:> sudo yum reinstall -y kernel || true
2024-04-10T22:39:14.930 INFO:teuthology.orchestra.run.smithi176.stdout:Updating Subscription Management repositories.
2024-04-10T22:39:14.930 INFO:teuthology.orchestra.run.smithi176.stdout:Unable to read consumer identity
2024-04-10T22:39:14.986 INFO:teuthology.orchestra.run.smithi176.stdout:
2024-04-10T22:39:14.986 INFO:teuthology.orchestra.run.smithi176.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register.
2024-04-10T22:39:14.986 INFO:teuthology.orchestra.run.smithi176.stdout:
2024-04-10T22:39:15.217 INFO:teuthology.orchestra.run.smithi118.stdout:Copr repo for python3-asyncssh owned by ceph    0.0  B/s |   0  B     00:45
2024-04-10T22:39:15.217 INFO:teuthology.orchestra.run.smithi118.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh':
2024-04-10T22:39:15.217 INFO:teuthology.orchestra.run.smithi118.stderr:  - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org]
2024-04-10T22:39:15.217 INFO:teuthology.orchestra.run.smithi118.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried
2024-04-10T22:39:15.367 INFO:teuthology.orchestra.run.smithi118.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh
2024-04-10T22:39:15.368 INFO:teuthology.orchestra.run.smithi118.stdout:Last metadata expiration check: 0:00:47 ago on Wed 10 Apr 2024 10:38:28 PM UTC.
2024-04-10T22:39:15.449 INFO:teuthology.orchestra.run.smithi118.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2024-04-10T22:39:15.450 INFO:teuthology.orchestra.run.smithi118.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available.
2024-04-10T22:39:15.491 INFO:teuthology.orchestra.run.smithi118.stderr:Error: No packages marked for reinstall.
2024-04-10T22:39:15.538 DEBUG:teuthology.orchestra.run.smithi118:> sudo yum reinstall -y kernel || true
2024-04-10T22:39:15.870 INFO:teuthology.orchestra.run.smithi118.stdout:Updating Subscription Management repositories.
2024-04-10T22:39:15.871 INFO:teuthology.orchestra.run.smithi118.stdout:Unable to read consumer identity
2024-04-10T22:39:15.928 INFO:teuthology.orchestra.run.smithi118.stdout:
2024-04-10T22:39:15.928 INFO:teuthology.orchestra.run.smithi118.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register.
2024-04-10T22:39:15.928 INFO:teuthology.orchestra.run.smithi118.stdout:
2024-04-10T22:39:55.458 INFO:teuthology.orchestra.run.smithi176.stdout:Copr repo for python3-asyncssh owned by ceph    0.0  B/s |   0  B     00:40
2024-04-10T22:39:55.458 INFO:teuthology.orchestra.run.smithi176.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh':
2024-04-10T22:39:55.459 INFO:teuthology.orchestra.run.smithi176.stderr:  - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org]
2024-04-10T22:39:55.459 INFO:teuthology.orchestra.run.smithi176.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried
2024-04-10T22:39:55.601 INFO:teuthology.orchestra.run.smithi118.stdout:Copr repo for python3-asyncssh owned by ceph    0.0  B/s |   0  B     00:39
2024-04-10T22:39:55.601 INFO:teuthology.orchestra.run.smithi118.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh':
2024-04-10T22:39:55.602 INFO:teuthology.orchestra.run.smithi118.stderr:  - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org]
2024-04-10T22:39:55.602 INFO:teuthology.orchestra.run.smithi118.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried
2024-04-10T22:39:55.614 INFO:teuthology.orchestra.run.smithi176.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh
2024-04-10T22:39:55.615 INFO:teuthology.orchestra.run.smithi176.stdout:Last metadata expiration check: 0:01:36 ago on Wed 10 Apr 2024 10:38:19 PM UTC.
2024-04-10T22:39:55.679 INFO:teuthology.orchestra.run.smithi176.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2024-04-10T22:39:55.680 INFO:teuthology.orchestra.run.smithi176.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available.
2024-04-10T22:39:55.714 INFO:teuthology.orchestra.run.smithi176.stderr:Error: No packages marked for reinstall.
2024-04-10T22:39:55.752 INFO:teuthology.orchestra.run.smithi118.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh
2024-04-10T22:39:55.752 INFO:teuthology.orchestra.run.smithi118.stdout:Last metadata expiration check: 0:01:27 ago on Wed 10 Apr 2024 10:38:28 PM UTC.
2024-04-10T22:39:55.764 DEBUG:teuthology.orchestra.run.smithi176:> rpm -q kernel | sort -rV | head -n 1
2024-04-10T22:39:55.814 INFO:teuthology.orchestra.run.smithi118.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2024-04-10T22:39:55.814 INFO:teuthology.orchestra.run.smithi118.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available.
2024-04-10T22:39:55.853 INFO:teuthology.orchestra.run.smithi118.stderr:Error: No packages marked for reinstall.
2024-04-10T22:39:55.904 DEBUG:teuthology.orchestra.run.smithi118:> rpm -q kernel | sort -rV | head -n 1
2024-04-10T22:39:55.921 INFO:teuthology.orchestra.run.smithi176.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:39:55.921 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:39:55.921 INFO:teuthology.task.kernel:Newest distro kernel installed and running
2024-04-10T22:39:55.921 INFO:teuthology.task.kernel:Enabling kdb on mon.b...
2024-04-10T22:39:55.922 DEBUG:teuthology.orchestra.run.smithi176:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2024-04-10T22:39:55.956 INFO:teuthology.orchestra.run.smithi176.stdout:ttyS1
2024-04-10T22:39:55.972 DEBUG:teuthology.parallel:result is None
2024-04-10T22:39:56.078 INFO:teuthology.orchestra.run.smithi118.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:39:56.078 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:39:56.079 INFO:teuthology.task.kernel:Newest distro kernel installed and running
2024-04-10T22:39:56.079 INFO:teuthology.task.kernel:Enabling kdb on mon.a...
2024-04-10T22:39:56.079 DEBUG:teuthology.orchestra.run.smithi118:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2024-04-10T22:39:56.112 INFO:teuthology.orchestra.run.smithi118.stdout:ttyS1
2024-04-10T22:39:56.127 DEBUG:teuthology.parallel:result is None
2024-04-10T22:39:56.128 INFO:teuthology.run_tasks:Running task internal.base...
2024-04-10T22:39:56.137 INFO:teuthology.task.internal:Creating test directory...
2024-04-10T22:39:56.138 DEBUG:teuthology.orchestra.run.smithi118:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2024-04-10T22:39:56.170 DEBUG:teuthology.orchestra.run.smithi176:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2024-04-10T22:39:56.187 INFO:teuthology.run_tasks:Running task internal.archive_upload...
2024-04-10T22:39:56.193 INFO:teuthology.run_tasks:Running task internal.archive...
2024-04-10T22:39:56.201 INFO:teuthology.task.internal:Creating archive directory...
2024-04-10T22:39:56.201 DEBUG:teuthology.orchestra.run.smithi118:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2024-04-10T22:39:56.228 DEBUG:teuthology.orchestra.run.smithi176:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2024-04-10T22:39:56.255 INFO:teuthology.run_tasks:Running task internal.coredump...
2024-04-10T22:39:56.261 INFO:teuthology.task.internal:Enabling coredump saving...
2024-04-10T22:39:56.262 DEBUG:teuthology.orchestra.run.smithi118:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2024-04-10T22:39:56.291 DEBUG:teuthology.orchestra.run.smithi176:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2024-04-10T22:39:56.323 INFO:teuthology.orchestra.run.smithi118.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-10T22:39:56.329 INFO:teuthology.orchestra.run.smithi176.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-10T22:39:56.333 INFO:teuthology.orchestra.run.smithi118.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-10T22:39:56.340 INFO:teuthology.orchestra.run.smithi176.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-10T22:39:56.342 INFO:teuthology.run_tasks:Running task internal.sudo...
2024-04-10T22:39:56.349 INFO:teuthology.task.internal:Configuring sudo...
2024-04-10T22:39:56.349 DEBUG:teuthology.orchestra.run.smithi118:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2024-04-10T22:39:56.377 DEBUG:teuthology.orchestra.run.smithi176:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2024-04-10T22:39:56.409 INFO:teuthology.run_tasks:Running task internal.syslog...
2024-04-10T22:39:56.419 INFO:teuthology.task.internal.syslog:Starting syslog monitoring...
2024-04-10T22:39:56.419 DEBUG:teuthology.orchestra.run.smithi118:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2024-04-10T22:39:56.444 DEBUG:teuthology.orchestra.run.smithi176:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2024-04-10T22:39:56.466 DEBUG:teuthology.orchestra.run.smithi118:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-10T22:39:56.541 DEBUG:teuthology.orchestra.run.smithi118:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-10T22:39:56.598 DEBUG:teuthology.orchestra.run.smithi118:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-10T22:39:56.682 DEBUG:teuthology.orchestra.run.smithi118:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-10T22:39:56.723 DEBUG:teuthology.orchestra.run.smithi118:> set -ex
2024-04-10T22:39:56.723 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2024-04-10T22:39:56.791 DEBUG:teuthology.orchestra.run.smithi176:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-10T22:39:56.835 DEBUG:teuthology.orchestra.run.smithi176:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-10T22:39:56.899 DEBUG:teuthology.orchestra.run.smithi176:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-10T22:39:56.943 DEBUG:teuthology.orchestra.run.smithi176:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-10T22:39:56.985 DEBUG:teuthology.orchestra.run.smithi176:> set -ex
2024-04-10T22:39:56.985 DEBUG:teuthology.orchestra.run.smithi176:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2024-04-10T22:39:57.051 DEBUG:teuthology.orchestra.run.smithi118:> sudo service rsyslog restart
2024-04-10T22:39:57.054 DEBUG:teuthology.orchestra.run.smithi176:> sudo service rsyslog restart
2024-04-10T22:39:57.104 INFO:teuthology.orchestra.run.smithi118.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-10T22:39:57.148 INFO:teuthology.orchestra.run.smithi176.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-10T22:39:57.539 INFO:teuthology.run_tasks:Running task internal.timer...
2024-04-10T22:39:57.547 INFO:teuthology.task.internal:Starting timer...
2024-04-10T22:39:57.547 INFO:teuthology.run_tasks:Running task pcp...
2024-04-10T22:39:57.595 INFO:teuthology.run_tasks:Running task selinux...
2024-04-10T22:39:57.603 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}
2024-04-10T22:39:57.604 DEBUG:teuthology.orchestra.run.smithi118:> sudo service auditd rotate
2024-04-10T22:39:57.679 INFO:teuthology.orchestra.run.smithi118.stdout:Rotating logs:
2024-04-10T22:39:57.681 DEBUG:teuthology.orchestra.run.smithi176:> sudo service auditd rotate
2024-04-10T22:39:57.739 INFO:teuthology.orchestra.run.smithi176.stdout:Rotating logs:
2024-04-10T22:39:57.741 DEBUG:teuthology.task.selinux:Getting current SELinux state
2024-04-10T22:39:57.741 DEBUG:teuthology.orchestra.run.smithi118:> /usr/sbin/getenforce
2024-04-10T22:39:57.764 INFO:teuthology.orchestra.run.smithi118.stdout:Permissive
2024-04-10T22:39:57.764 DEBUG:teuthology.orchestra.run.smithi176:> /usr/sbin/getenforce
2024-04-10T22:39:57.786 INFO:teuthology.orchestra.run.smithi176.stdout:Permissive
2024-04-10T22:39:57.786 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi118.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi176.front.sepia.ceph.com': 'permissive'}
2024-04-10T22:39:57.786 DEBUG:teuthology.orchestra.run.smithi118:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2024-04-10T22:39:57.833 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:39:57.834 DEBUG:teuthology.orchestra.run.smithi176:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2024-04-10T22:39:57.863 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:39:57.863 INFO:teuthology.task.selinux:Putting SELinux into permissive mode
2024-04-10T22:39:57.863 DEBUG:teuthology.orchestra.run.smithi118:> sudo /usr/sbin/setenforce permissive
2024-04-10T22:39:57.926 DEBUG:teuthology.orchestra.run.smithi176:> sudo /usr/sbin/setenforce permissive
2024-04-10T22:39:57.967 INFO:teuthology.run_tasks:Running task ansible.cephlab...
2024-04-10T22:39:57.976 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main
2024-04-10T22:39:57.992 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}]
2024-04-10T22:39:57.993 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi118.front.sepia.ceph.com,smithi176.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml
2024-04-10T22:56:41.445 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650186/ansible_failures.yaml
2024-04-10T22:56:41.447 ERROR:teuthology.run_tasks:Saw exception from tasks.
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin
    super(CephLab, self).begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin
    self.execute_playbook()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook
    self._handle_failure(command, status)
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure
    raise AnsibleFailedError(failures)
teuthology.exceptions.AnsibleFailedError: ['<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at']
2024-04-10T22:56:41.676 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=c07495c63f0342cbba94db989209bbc2
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin
    super(CephLab, self).begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin
    self.execute_playbook()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook
    self._handle_failure(command, status)
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure
    raise AnsibleFailedError(failures)
teuthology.exceptions.AnsibleFailedError: ['<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at']
2024-04-10T22:56:41.678 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab
2024-04-10T22:56:41.686 INFO:teuthology.task.ansible:Skipping ansible cleanup...
2024-04-10T22:56:41.687 DEBUG:teuthology.run_tasks:Unwinding manager selinux
2024-04-10T22:56:41.697 DEBUG:teuthology.orchestra.run.smithi118:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log
2024-04-10T22:56:41.779 DEBUG:teuthology.orchestra.run.smithi176:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log
2024-04-10T22:56:41.844 DEBUG:teuthology.orchestra.run.smithi118:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2024-04-10T22:56:41.874 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:56:41.875 DEBUG:teuthology.orchestra.run.smithi176:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2024-04-10T22:56:41.905 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:56:41.906 DEBUG:teuthology.run_tasks:Unwinding manager pcp
2024-04-10T22:56:41.915 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer
2024-04-10T22:56:41.923 INFO:teuthology.task.internal:Duration was 1004.375612 seconds
2024-04-10T22:56:41.923 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog
2024-04-10T22:56:41.931 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring...
2024-04-10T22:56:41.931 DEBUG:teuthology.orchestra.run.smithi118:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart
2024-04-10T22:56:41.933 DEBUG:teuthology.orchestra.run.smithi176:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart
2024-04-10T22:56:41.982 INFO:teuthology.orchestra.run.smithi118.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-10T22:56:41.995 INFO:teuthology.orchestra.run.smithi176.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-10T22:56:42.300 INFO:teuthology.task.internal.syslog:Checking logs for errors...
2024-04-10T22:56:42.300 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi118.front.sepia.ceph.com
2024-04-10T22:56:42.300 DEBUG:teuthology.orchestra.run.smithi118:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1
2024-04-10T22:56:42.324 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi176.front.sepia.ceph.com
2024-04-10T22:56:42.324 DEBUG:teuthology.orchestra.run.smithi176:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1
2024-04-10T22:56:42.346 INFO:teuthology.task.internal.syslog:Compressing syslogs...
2024-04-10T22:56:42.347 DEBUG:teuthology.orchestra.run.smithi118:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
2024-04-10T22:56:42.366 DEBUG:teuthology.orchestra.run.smithi176:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
2024-04-10T22:56:42.428 INFO:teuthology.task.internal.syslog:Gathering journactl -b0...
2024-04-10T22:56:42.428 DEBUG:teuthology.orchestra.run.smithi118:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz
2024-04-10T22:56:42.449 DEBUG:teuthology.orchestra.run.smithi176:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz
2024-04-10T22:56:42.535 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo
2024-04-10T22:56:42.545 INFO:teuthology.task.internal:Restoring /etc/sudoers...
2024-04-10T22:56:42.545 DEBUG:teuthology.orchestra.run.smithi118:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers
2024-04-10T22:56:42.572 DEBUG:teuthology.orchestra.run.smithi176:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers
2024-04-10T22:56:42.599 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
2024-04-10T22:56:42.609 DEBUG:teuthology.orchestra.run.smithi118:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
2024-04-10T22:56:42.615 DEBUG:teuthology.orchestra.run.smithi176:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
2024-04-10T22:56:42.639 INFO:teuthology.orchestra.run.smithi118.stdout:kernel.core_pattern = core
2024-04-10T22:56:42.665 INFO:teuthology.orchestra.run.smithi176.stdout:kernel.core_pattern = core
2024-04-10T22:56:42.688 DEBUG:teuthology.orchestra.run.smithi118:> test -e /home/ubuntu/cephtest/archive/coredump
2024-04-10T22:56:42.743 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:56:42.744 DEBUG:teuthology.orchestra.run.smithi176:> test -e /home/ubuntu/cephtest/archive/coredump
2024-04-10T22:56:42.759 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:56:42.760 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive
2024-04-10T22:56:42.770 INFO:teuthology.task.internal:Transferring archived files...
2024-04-10T22:56:42.772 DEBUG:teuthology.misc:Transferring archived files from smithi118:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650186/remote/smithi118
2024-04-10T22:56:42.773 DEBUG:teuthology.orchestra.run.smithi118:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- .
2024-04-10T22:56:42.857 DEBUG:teuthology.misc:Transferring archived files from smithi176:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650186/remote/smithi176
2024-04-10T22:56:42.858 DEBUG:teuthology.orchestra.run.smithi176:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- .
2024-04-10T22:56:42.912 INFO:teuthology.task.internal:Removing archive directory...
2024-04-10T22:56:42.913 DEBUG:teuthology.orchestra.run.smithi118:> rm -rf -- /home/ubuntu/cephtest/archive
2024-04-10T22:56:42.915 DEBUG:teuthology.orchestra.run.smithi176:> rm -rf -- /home/ubuntu/cephtest/archive
2024-04-10T22:56:42.932 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload
2024-04-10T22:56:42.978 INFO:teuthology.task.internal:Not uploading archives.
2024-04-10T22:56:42.979 DEBUG:teuthology.run_tasks:Unwinding manager internal.base
2024-04-10T22:56:42.987 INFO:teuthology.task.internal:Tidying up after the test...
2024-04-10T22:56:42.987 DEBUG:teuthology.orchestra.run.smithi118:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest
2024-04-10T22:56:42.989 DEBUG:teuthology.orchestra.run.smithi176:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest
2024-04-10T22:56:43.003 INFO:teuthology.orchestra.run.smithi118.stdout:   262171      4 drwxr-xr-x   2  ubuntu   ubuntu       4096 Apr 10 22:56 /home/ubuntu/cephtest
2024-04-10T22:56:43.006 INFO:teuthology.orchestra.run.smithi176.stdout:   262171      4 drwxr-xr-x   2  ubuntu   ubuntu       4096 Apr 10 22:56 /home/ubuntu/cephtest
2024-04-10T22:56:43.007 DEBUG:teuthology.run_tasks:Unwinding manager kernel
2024-04-10T22:56:43.016 DEBUG:teuthology.run_tasks:Unwinding manager console_log
2024-04-10T22:56:43.034 INFO:teuthology.nuke:Checking targets against current locks
2024-04-10T22:56:43.074 DEBUG:teuthology.nuke:shortname: smithi118
2024-04-10T22:56:43.075 INFO:teuthology.task.internal.check_lock:Checking locks...
2024-04-10T22:56:43.119 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi118.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650186', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:17.965214', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP9+bjT+d23cXpNHyWMczgMDvi2F+N89PzP8dIT+PvPP+mZ7zjds37K3I3l1T1rSKFATCEvqTOShWlr8GC4qDSY='}
2024-04-10T22:56:43.133 DEBUG:teuthology.nuke:shortname: smithi176
2024-04-10T22:56:43.134 INFO:teuthology.task.internal.check_lock:Checking locks...
2024-04-10T22:56:43.164 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi176.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650186', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:17.963494', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLloNgZVPx4NOZ6z6LwGKk2JoLLrAgyLHbfDjHK88gkCj5k10tOApVyo2RVha6z1X3iqkcVJrMWZl3ZuiJxoHNo='}
2024-04-10T22:56:43.230 INFO:teuthology.orchestra.console.smithi118:Power off
2024-04-10T22:56:43.230 DEBUG:teuthology.orchestra.console.smithi118:pexpect command: ipmitool -H smithi118.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2024-04-10T22:56:43.264 INFO:teuthology.orchestra.console.smithi176:Power off
2024-04-10T22:56:43.265 DEBUG:teuthology.orchestra.console.smithi176:pexpect command: ipmitool -H smithi176.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2024-04-10T22:56:43.283 DEBUG:teuthology.orchestra.console.smithi118:power off output: Chassis Power Control: Down/Off
2024-04-10T22:56:43.284 DEBUG:teuthology.orchestra.console.smithi118:pexpect command: ipmitool -H smithi118.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-10T22:56:43.317 DEBUG:teuthology.orchestra.console.smithi176:power off output: Chassis Power Control: Down/Off
2024-04-10T22:56:43.317 DEBUG:teuthology.orchestra.console.smithi176:pexpect command: ipmitool -H smithi176.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-10T22:56:43.350 DEBUG:teuthology.orchestra.console.smithi118:check power output: Chassis Power is on
2024-04-10T22:56:43.371 DEBUG:teuthology.orchestra.console.smithi176:check power output: Chassis Power is on
2024-04-10T22:56:47.351 DEBUG:teuthology.orchestra.console.smithi118:pexpect command: ipmitool -H smithi118.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-10T22:56:47.371 DEBUG:teuthology.orchestra.console.smithi176:pexpect command: ipmitool -H smithi176.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-10T22:56:47.462 DEBUG:teuthology.orchestra.console.smithi118:check power output: Chassis Power is on
2024-04-10T22:56:47.482 DEBUG:teuthology.orchestra.console.smithi176:check power output: Chassis Power is on
2024-04-10T22:56:51.467 DEBUG:teuthology.orchestra.console.smithi118:pexpect command: ipmitool -H smithi118.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-10T22:56:51.483 DEBUG:teuthology.orchestra.console.smithi176:pexpect command: ipmitool -H smithi176.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-10T22:56:51.578 DEBUG:teuthology.orchestra.console.smithi118:check power output: Chassis Power is off
2024-04-10T22:56:51.594 DEBUG:teuthology.orchestra.console.smithi176:check power output: Chassis Power is off
2024-04-10T22:56:51.679 INFO:teuthology.orchestra.console.smithi118:Power off completed
2024-04-10T22:56:51.695 INFO:teuthology.orchestra.console.smithi176:Power off completed
2024-04-10T22:56:51.801 INFO:teuthology.run:Summary data:
description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8
  0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls
  rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench
  mon_election/classic}
duration: 1004.375611782074
failure_reason: '[''<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,'',
  ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling
  a Copr repository. Please note that this repository is not part'', ''Error: Failed
  to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64:
  Network is unreachable'', ''Please do not file bug reports about these packages
  in Fedora'', ''The Fedora Project does not exercise any power over the contents
  of'', ''and packages are not held to any quality or security level.'', ''of the
  main distribution, and quality may vary.'', ''this repository beyond the rules outlined
  in the Copr FAQ at'']'
owner: scheduled_teuthology@teuthology
sentry_event: https://sentry.ceph.com/organizations/ceph/?query=c07495c63f0342cbba94db989209bbc2
status: dead
success: false

2024-04-10T22:56:51.801 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/
2024-04-10T22:56:51.914 INFO:teuthology.run:DEAD