2024-04-10T22:37:12.729 INFO:root:teuthology version: 0.0.1.dev267+g6c63784
2024-04-10T22:37:12.729 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650180 --verbose --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650180/orig.config.yaml
2024-04-10T22:37:12.788 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/
2024-04-10T22:37:12.878 INFO:teuthology.run:Config:
  archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650180
  branch: reef
  description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0
    0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls
    rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench
    mon_election/classic}
  email: ceph-qa@ceph.com
  first-half-sequence:
  - cephadm.shell:
      env:
      - sha1
      mon.a:
      - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60
      - ceph config set global log_to_journald false --force
      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
      - ceph orch ps
      - echo wait for minority of mons to upgrade
      - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done
      - ceph orch ps
      - ceph orch upgrade pause
      - sleep 60
      - ceph orch upgrade resume
      - echo wait for majority of mons to upgrade
      - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done'
      - ceph orch ps
      - ceph orch upgrade pause
      - sleep 60
      - ceph orch upgrade resume
      - echo wait for all mons to upgrade
      - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done'
      - ceph orch ps
      - ceph orch upgrade pause
      - sleep 60
      - ceph orch upgrade resume
      - echo wait for half of osds to upgrade
      - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ;
        done'
      - ceph orch upgrade pause
      - ceph orch ps
      - ceph orch ps
      - ceph versions
  first-half-tasks:
  - full_sequential:
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
  - print: '**** done end radosbench.yaml'
  first_in_suite: false
  job_id: '7650180'
  kernel:
    kdb: 1
    sha1: distro
  last_in_suite: false
  machine_type: smithi
  meta:
  - desc: 'run randomized correctness test for rados operations
  
      generate write load with rados bench
  
      '
  - desc: 'run randomized correctness test for rados operations
  
      generate write load with rados bench
  
      '
  - desc: 'run basic cls tests for rbd
  
      '
  - desc: 'run basic import/export cli tests for rbd
  
      '
  - desc: 'librbd C and C++ api tests
  
      '
  - desc: 'randomized correctness test for rados operations on a replicated pool,
  
      using only reads, writes, and deletes
  
      '
  - desc: 'randomized correctness test for rados operations on a replicated pool with
      snapshot operations
  
      '
  - desc: 'run randomized correctness test for rados operations
  
      generate write load with rados bench
  
      '
  name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi
  no_nested_subset: false
  nuke-on-error: true
  openstack:
  - volumes:
      count: 4
      size: 10
  os_type: rhel
  os_version: '8.6'
  overrides:
    admin_socket:
      branch: reef
    ceph:
      conf:
        global:
          mon election default strategy: 1
        mgr:
          debug mgr: 20
          debug ms: 1
        mon:
          debug mon: 20
          debug ms: 1
          debug paxos: 20
        osd:
          debug ms: 1
          debug osd: 20
          osd shutdown pgref assert: true
      create_rbd_pool: true
      flavor: default
      log-ignorelist:
      - \(MDS_ALL_DOWN\)
      - \(MDS_UP_LESS_THAN_MAX\)
      sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42
    ceph-deploy:
      conf:
        client:
          log file: /var/log/ceph/ceph-$name.$pid.log
        mon: {}
    install:
      ceph:
        flavor: default
        sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42
    selinux:
      whitelist:
      - scontext=system_u:system_r:logrotate_t:s0
    workunit:
      branch: reef
      sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c
  owner: scheduled_teuthology@teuthology
  priority: 830
  repo: https://git.ceph.com/ceph.git
  roles:
  - - mon.a
    - mon.c
    - mgr.y
    - osd.0
    - osd.1
    - osd.2
    - osd.3
    - client.0
    - node-exporter.a
    - alertmanager.a
  - - mon.b
    - mgr.x
    - osd.4
    - osd.5
    - osd.6
    - osd.7
    - client.1
    - prometheus.a
    - grafana.a
    - node-exporter.b
  second-half-sequence:
    sequential:
    - cephadm.shell:
        env:
        - sha1
        mon.a:
        - ceph orch upgrade resume
        - sleep 60
        - echo wait for upgrade to complete
        - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch
          ps ; ceph versions ; sleep 30 ; done
        - echo upgrade complete
        - ceph orch ps
        - ceph versions
        - ceph versions | jq -e '.overall | length == 1'
        - ceph versions | jq -e '.overall | keys' | grep $sha1
  second-half-tasks:
  - full_sequential:
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
  - print: '**** done end radosbench.yaml'
  seed: 4604
  sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42
  sleep_before_teardown: 0
  stress-tasks:
  - thrashosds:
      aggressive_pg_num_changes: false
      chance_force_recovery: 0
      chance_pgnum_grow: 1
      chance_pgpnum_fix: 1
      chance_thrash_cluster_full: 0
      chance_thrash_pg_upmap: 0
      chance_thrash_pg_upmap_items: 0
      disable_objectstore_tool_tests: true
      timeout: 1200
  - full_sequential:
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
    - radosbench:
        clients:
        - client.0
        time: 90
  - print: '**** done end radosbench.yaml'
  - workunit:
      branch: pacific
      clients:
        client.0:
        - cls/test_cls_rbd.sh
  - print: '**** done cls/test_cls_rbd.sh 5-workload'
  - workunit:
      branch: pacific
      clients:
        client.0:
        - rbd/import_export.sh
      env:
        RBD_CREATE_ARGS: --new-format
  - print: '**** done rbd/import_export.sh 5-workload'
  - workunit:
      branch: pacific
      clients:
        client.0:
        - rbd/test_librbd.sh
      env:
        RBD_FEATURES: '61'
  - print: '**** done rbd/test_librbd.sh 7-workload'
  - full_sequential:
    - rados:
        clients:
        - client.0
        objects: 500
        op_weights:
          delete: 10
          read: 45
          write: 45
        ops: 4000
        write_append_excl: false
  - print: '**** done rados/readwrite 5-workload'
  - full_sequential:
    - rados:
        clients:
        - client.0
        objects: 50
        op_weights:
          delete: 50
          read: 100
          rollback: 50
          snap_create: 50
          snap_remove: 50
          write: 100
        ops: 4000
        write_append_excl: false
  - print: '**** done rados/snaps-few-objects 5-workload'
  subset: 0/1
  suite: upgrade:pacific-x
  suite_branch: reef
  suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa
  suite_relpath: qa
  suite_repo: https://git.ceph.com/ceph.git
  suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c
  targets:
    smithi071.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMsKWubXofcVlZXZdH0ffoh6ztUo2wLmmZF1dA/7QzvD7Z+aFib5ONAt+U28A6pXyl8EDWj3rCv/PX/D7OvRf5s=
    smithi183.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC+OQVzPG1Sf1eQr1uHKpf6a/BJXRAPQQRrpAgNFKmwthQhRYAo2vZ2P1tsw5aUj6NvbWxHZ8xeID9yLtf+lw7g=
  tasks:
  - pexec:
      all:
      - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
      - sudo dnf -y  module reset container-tools
      - sudo dnf -y  module install container-tools:3.0 --allowerasing --nobest
      - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
  - install:
      branch: pacific
      exclude_packages:
      - ceph-volume
  - cephadm:
      cephadm_branch: pacific
      cephadm_git_url: https://github.com/ceph/ceph
      conf:
        osd:
          osd_class_default_list: '*'
          osd_class_load_list: '*'
      image: quay.ceph.io/ceph-ci/ceph:pacific
  - cephadm.shell:
      mon.a:
      - ceph fs volume create foo
      - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
      - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
  - ceph.healthy: null
  - print: '**** upgrading first half of cluster, with stress ****'
  - parallel:
    - first-half-tasks
    - first-half-sequence
  - print: '**** done upgrading first half of cluster ****'
  - ceph.healthy: null
  - print: '**** applying stress + thrashing to mixed-version cluster ****'
  - parallel:
    - stress-tasks
  - ceph.healthy: null
  - print: '**** finishing upgrade ****'
  - parallel:
    - second-half-tasks
    - second-half-sequence
  - ceph.healthy: null
  teuthology:
    fragments_dropped: []
    meta: {}
    postmerge: []
  teuthology_branch: main
  teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab
  timestamp: 2024-04-10_01:08:08
  tube: smithi
  user: teuthology
  verbose: false
  worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885
2024-04-10T22:37:12.878 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it
2024-04-10T22:37:12.878 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks
2024-04-10T22:37:12.879 INFO:teuthology.run_tasks:Running task internal.check_packages...
2024-04-10T22:37:12.880 INFO:teuthology.task.internal:Checking packages...
2024-04-10T22:37:12.902 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42'
2024-04-10T22:37:12.902 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch
2024-04-10T22:37:12.902 INFO:teuthology.packaging:ref: None
2024-04-10T22:37:12.902 INFO:teuthology.packaging:tag: None
2024-04-10T22:37:12.902 INFO:teuthology.packaging:branch: reef
2024-04-10T22:37:12.902 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42
2024-04-10T22:37:12.902 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef
2024-04-10T22:37:13.236 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47
2024-04-10T22:37:13.237 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep...
2024-04-10T22:37:13.247 INFO:teuthology.task.internal:no buildpackages task found
2024-04-10T22:37:13.248 INFO:teuthology.run_tasks:Running task internal.save_config...
2024-04-10T22:37:13.290 INFO:teuthology.task.internal:Saving configuration
2024-04-10T22:37:13.311 INFO:teuthology.run_tasks:Running task internal.check_lock...
2024-04-10T22:37:13.317 INFO:teuthology.task.internal.check_lock:Checking locks...
2024-04-10T22:37:13.341 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi071.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650180', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:15.479212', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMsKWubXofcVlZXZdH0ffoh6ztUo2wLmmZF1dA/7QzvD7Z+aFib5ONAt+U28A6pXyl8EDWj3rCv/PX/D7OvRf5s='}
2024-04-10T22:37:13.358 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi183.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650180', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:15.480655', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC+OQVzPG1Sf1eQr1uHKpf6a/BJXRAPQQRrpAgNFKmwthQhRYAo2vZ2P1tsw5aUj6NvbWxHZ8xeID9yLtf+lw7g='}
2024-04-10T22:37:13.358 INFO:teuthology.run_tasks:Running task internal.add_remotes...
2024-04-10T22:37:13.365 INFO:teuthology.task.internal:roles: ubuntu@smithi071.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a']
2024-04-10T22:37:13.365 INFO:teuthology.task.internal:roles: ubuntu@smithi183.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b']
2024-04-10T22:37:13.365 INFO:teuthology.run_tasks:Running task console_log...
2024-04-10T22:37:13.457 DEBUG:teuthology.exit:Installing handler: Handler(exiter=<teuthology.exit.Exiter object at 0x7fba145f6790>, func=<function ConsoleLog.start_logging.<locals>.kill_console_loggers at 0x7fba153df0d0>, signals=[15])
2024-04-10T22:37:13.458 INFO:teuthology.run_tasks:Running task internal.connect...
2024-04-10T22:37:13.464 INFO:teuthology.task.internal:Opening connections...
2024-04-10T22:37:13.464 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi071.front.sepia.ceph.com
2024-04-10T22:37:13.466 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi071.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-10T22:37:13.539 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi183.front.sepia.ceph.com
2024-04-10T22:37:13.540 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi183.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60}
2024-04-10T22:37:13.613 INFO:teuthology.run_tasks:Running task internal.push_inventory...
2024-04-10T22:37:13.620 DEBUG:teuthology.orchestra.run.smithi071:> uname -m
2024-04-10T22:37:13.639 INFO:teuthology.orchestra.run.smithi071.stdout:x86_64
2024-04-10T22:37:13.640 DEBUG:teuthology.orchestra.run.smithi071:> cat /etc/os-release
2024-04-10T22:37:13.698 INFO:teuthology.orchestra.run.smithi071.stdout:NAME="Red Hat Enterprise Linux"
2024-04-10T22:37:13.698 INFO:teuthology.orchestra.run.smithi071.stdout:VERSION="8.6 (Ootpa)"
2024-04-10T22:37:13.698 INFO:teuthology.orchestra.run.smithi071.stdout:ID="rhel"
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:ID_LIKE="fedora"
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:VERSION_ID="8.6"
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:PLATFORM_ID="platform:el8"
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)"
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:ANSI_COLOR="0;31"
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos"
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:HOME_URL="https://www.redhat.com/"
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/"
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/"
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8"
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
2024-04-10T22:37:13.699 INFO:teuthology.orchestra.run.smithi071.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6"
2024-04-10T22:37:13.700 INFO:teuthology.lock.ops:Updating smithi071.front.sepia.ceph.com on lock server
2024-04-10T22:37:13.721 DEBUG:teuthology.orchestra.run.smithi183:> uname -m
2024-04-10T22:37:13.740 INFO:teuthology.orchestra.run.smithi183.stdout:x86_64
2024-04-10T22:37:13.740 DEBUG:teuthology.orchestra.run.smithi183:> cat /etc/os-release
2024-04-10T22:37:13.799 INFO:teuthology.orchestra.run.smithi183.stdout:NAME="Red Hat Enterprise Linux"
2024-04-10T22:37:13.799 INFO:teuthology.orchestra.run.smithi183.stdout:VERSION="8.6 (Ootpa)"
2024-04-10T22:37:13.799 INFO:teuthology.orchestra.run.smithi183.stdout:ID="rhel"
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:ID_LIKE="fedora"
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:VERSION_ID="8.6"
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:PLATFORM_ID="platform:el8"
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)"
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:ANSI_COLOR="0;31"
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos"
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:HOME_URL="https://www.redhat.com/"
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/"
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/"
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8"
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
2024-04-10T22:37:13.800 INFO:teuthology.orchestra.run.smithi183.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6"
2024-04-10T22:37:13.801 INFO:teuthology.lock.ops:Updating smithi183.front.sepia.ceph.com on lock server
2024-04-10T22:37:13.829 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles...
2024-04-10T22:37:13.841 INFO:teuthology.run_tasks:Running task internal.check_conflict...
2024-04-10T22:37:13.865 INFO:teuthology.task.internal:Checking for old test directory...
2024-04-10T22:37:13.866 DEBUG:teuthology.orchestra.run.smithi071:> test '!' -e /home/ubuntu/cephtest
2024-04-10T22:37:13.868 DEBUG:teuthology.orchestra.run.smithi183:> test '!' -e /home/ubuntu/cephtest
2024-04-10T22:37:13.887 INFO:teuthology.run_tasks:Running task internal.check_ceph_data...
2024-04-10T22:37:13.893 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph...
2024-04-10T22:37:13.893 DEBUG:teuthology.orchestra.run.smithi071:> test -z $(ls -A /var/lib/ceph)
2024-04-10T22:37:13.927 DEBUG:teuthology.orchestra.run.smithi183:> test -z $(ls -A /var/lib/ceph)
2024-04-10T22:37:13.956 INFO:teuthology.run_tasks:Running task internal.vm_setup...
2024-04-10T22:37:14.008 INFO:teuthology.run_tasks:Running task kernel...
2024-04-10T22:37:14.022 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'}
2024-04-10T22:37:14.022 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300
2024-04-10T22:37:14.022 DEBUG:teuthology.orchestra.run.smithi071:> test -f /run/.containerenv -o -f /.dockerenv
2024-04-10T22:37:14.023 DEBUG:teuthology.orchestra.run.smithi183:> test -f /run/.containerenv -o -f /.dockerenv
2024-04-10T22:37:14.041 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:37:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:37:14.042 DEBUG:teuthology.orchestra.run.smithi071:> uname -r
2024-04-10T22:37:14.042 DEBUG:teuthology.orchestra.run.smithi183:> uname -r
2024-04-10T22:37:14.102 INFO:teuthology.orchestra.run.smithi183.stdout:4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:37:14.102 INFO:teuthology.task.kernel:Running kernel on smithi183: 4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:37:14.102 DEBUG:teuthology.orchestra.run.smithi183:> sudo yum install -y kernel
2024-04-10T22:37:14.103 INFO:teuthology.orchestra.run.smithi071.stdout:4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:37:14.103 INFO:teuthology.task.kernel:Running kernel on smithi071: 4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:37:14.103 DEBUG:teuthology.orchestra.run.smithi071:> sudo yum install -y kernel
2024-04-10T22:37:15.559 INFO:teuthology.orchestra.run.smithi183.stdout:Updating Subscription Management repositories.
2024-04-10T22:37:15.559 INFO:teuthology.orchestra.run.smithi183.stdout:Unable to read consumer identity
2024-04-10T22:37:15.567 INFO:teuthology.orchestra.run.smithi071.stdout:Updating Subscription Management repositories.
2024-04-10T22:37:15.567 INFO:teuthology.orchestra.run.smithi071.stdout:Unable to read consumer identity
2024-04-10T22:37:15.659 INFO:teuthology.orchestra.run.smithi183.stdout:
2024-04-10T22:37:15.659 INFO:teuthology.orchestra.run.smithi183.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register.
2024-04-10T22:37:15.659 INFO:teuthology.orchestra.run.smithi183.stdout:
2024-04-10T22:37:15.680 INFO:teuthology.orchestra.run.smithi071.stdout:
2024-04-10T22:37:15.680 INFO:teuthology.orchestra.run.smithi071.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register.
2024-04-10T22:37:15.680 INFO:teuthology.orchestra.run.smithi071.stdout:
2024-04-10T22:37:54.611 INFO:teuthology.orchestra.run.smithi183.stdout:Copr repo for python3-asyncssh owned by ceph    0.0  B/s |   0  B     00:37
2024-04-10T22:37:54.612 INFO:teuthology.orchestra.run.smithi183.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh':
2024-04-10T22:37:54.612 INFO:teuthology.orchestra.run.smithi183.stderr:  - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org]
2024-04-10T22:37:54.612 INFO:teuthology.orchestra.run.smithi183.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried
2024-04-10T22:37:54.697 INFO:teuthology.orchestra.run.smithi071.stdout:Copr repo for python3-asyncssh owned by ceph    0.0  B/s |   0  B     00:37
2024-04-10T22:37:54.698 INFO:teuthology.orchestra.run.smithi071.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh':
2024-04-10T22:37:54.698 INFO:teuthology.orchestra.run.smithi071.stderr:  - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org]
2024-04-10T22:37:54.698 INFO:teuthology.orchestra.run.smithi071.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried
2024-04-10T22:37:56.395 INFO:teuthology.orchestra.run.smithi071.stdout:Extra Packages for Enterprise Linux             9.6 MB/s |  16 MB     00:01
2024-04-10T22:37:59.978 INFO:teuthology.orchestra.run.smithi071.stdout:lab-extras                                      263 kB/s |  24 kB     00:00
2024-04-10T22:38:00.051 INFO:teuthology.orchestra.run.smithi071.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh
2024-04-10T22:38:00.051 INFO:teuthology.orchestra.run.smithi071.stdout:Last metadata expiration check: 0:00:01 ago on Wed 10 Apr 2024 10:37:59 PM UTC.
2024-04-10T22:38:00.697 INFO:teuthology.orchestra.run.smithi071.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed.
2024-04-10T22:38:00.697 INFO:teuthology.orchestra.run.smithi071.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed.
2024-04-10T22:38:00.770 INFO:teuthology.orchestra.run.smithi071.stdout:Dependencies resolved.
2024-04-10T22:38:00.772 INFO:teuthology.orchestra.run.smithi071.stdout:Nothing to do.
2024-04-10T22:38:00.772 INFO:teuthology.orchestra.run.smithi071.stdout:Complete!
2024-04-10T22:38:00.902 DEBUG:teuthology.orchestra.run.smithi071:> echo no | sudo yum reinstall kernel || true
2024-04-10T22:38:01.239 INFO:teuthology.orchestra.run.smithi071.stdout:Updating Subscription Management repositories.
2024-04-10T22:38:01.240 INFO:teuthology.orchestra.run.smithi071.stdout:Unable to read consumer identity
2024-04-10T22:38:01.295 INFO:teuthology.orchestra.run.smithi071.stdout:
2024-04-10T22:38:01.295 INFO:teuthology.orchestra.run.smithi071.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register.
2024-04-10T22:38:01.295 INFO:teuthology.orchestra.run.smithi071.stdout:
2024-04-10T22:38:44.969 INFO:teuthology.orchestra.run.smithi071.stdout:Copr repo for python3-asyncssh owned by ceph    0.0  B/s |   0  B     00:43
2024-04-10T22:38:44.970 INFO:teuthology.orchestra.run.smithi071.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh':
2024-04-10T22:38:44.970 INFO:teuthology.orchestra.run.smithi071.stderr:  - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org]
2024-04-10T22:38:44.970 INFO:teuthology.orchestra.run.smithi071.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried
2024-04-10T22:38:45.124 INFO:teuthology.orchestra.run.smithi071.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh
2024-04-10T22:38:45.125 INFO:teuthology.orchestra.run.smithi071.stdout:Last metadata expiration check: 0:00:46 ago on Wed 10 Apr 2024 10:37:59 PM UTC.
2024-04-10T22:38:45.225 INFO:teuthology.orchestra.run.smithi071.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2024-04-10T22:38:45.225 INFO:teuthology.orchestra.run.smithi071.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available.
2024-04-10T22:38:45.270 INFO:teuthology.orchestra.run.smithi071.stderr:Error: No packages marked for reinstall.
2024-04-10T22:38:45.316 DEBUG:teuthology.orchestra.run.smithi071:> sudo yum reinstall -y kernel || true
2024-04-10T22:38:45.650 INFO:teuthology.orchestra.run.smithi071.stdout:Updating Subscription Management repositories.
2024-04-10T22:38:45.650 INFO:teuthology.orchestra.run.smithi071.stdout:Unable to read consumer identity
2024-04-10T22:38:45.706 INFO:teuthology.orchestra.run.smithi071.stdout:
2024-04-10T22:38:45.706 INFO:teuthology.orchestra.run.smithi071.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register.
2024-04-10T22:38:45.706 INFO:teuthology.orchestra.run.smithi071.stdout:
2024-04-10T22:39:25.372 INFO:teuthology.orchestra.run.smithi071.stdout:Copr repo for python3-asyncssh owned by ceph    0.0  B/s |   0  B     00:39
2024-04-10T22:39:25.373 INFO:teuthology.orchestra.run.smithi071.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh':
2024-04-10T22:39:25.373 INFO:teuthology.orchestra.run.smithi071.stderr:  - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org]
2024-04-10T22:39:25.373 INFO:teuthology.orchestra.run.smithi071.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried
2024-04-10T22:39:25.516 INFO:teuthology.orchestra.run.smithi071.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh
2024-04-10T22:39:25.517 INFO:teuthology.orchestra.run.smithi071.stdout:Last metadata expiration check: 0:01:26 ago on Wed 10 Apr 2024 10:37:59 PM UTC.
2024-04-10T22:39:25.578 INFO:teuthology.orchestra.run.smithi071.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2024-04-10T22:39:25.578 INFO:teuthology.orchestra.run.smithi071.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available.
2024-04-10T22:39:25.609 INFO:teuthology.orchestra.run.smithi071.stderr:Error: No packages marked for reinstall.
2024-04-10T22:39:25.655 DEBUG:teuthology.orchestra.run.smithi071:> rpm -q kernel | sort -rV | head -n 1
2024-04-10T22:39:25.813 INFO:teuthology.orchestra.run.smithi071.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:39:25.813 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:39:25.814 INFO:teuthology.task.kernel:Newest distro kernel installed and running
2024-04-10T22:39:25.814 INFO:teuthology.task.kernel:Enabling kdb on mon.a...
2024-04-10T22:39:25.814 DEBUG:teuthology.orchestra.run.smithi071:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2024-04-10T22:39:25.845 INFO:teuthology.orchestra.run.smithi071.stdout:ttyS1
2024-04-10T22:39:25.861 DEBUG:teuthology.parallel:result is None
2024-04-10T22:40:12.065 INFO:teuthology.orchestra.run.smithi183.stdout:Extra Packages for Enterprise Linux             121 kB/s |  16 MB     02:17
2024-04-10T22:40:15.686 INFO:teuthology.orchestra.run.smithi183.stdout:lab-extras                                      538 kB/s |  24 kB     00:00
2024-04-10T22:40:15.760 INFO:teuthology.orchestra.run.smithi183.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh
2024-04-10T22:40:16.398 INFO:teuthology.orchestra.run.smithi183.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed.
2024-04-10T22:40:16.398 INFO:teuthology.orchestra.run.smithi183.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed.
2024-04-10T22:40:16.487 INFO:teuthology.orchestra.run.smithi183.stdout:Dependencies resolved.
2024-04-10T22:40:16.488 INFO:teuthology.orchestra.run.smithi183.stdout:Nothing to do.
2024-04-10T22:40:16.488 INFO:teuthology.orchestra.run.smithi183.stdout:Complete!
2024-04-10T22:40:16.604 DEBUG:teuthology.orchestra.run.smithi183:> echo no | sudo yum reinstall kernel || true
2024-04-10T22:40:16.982 INFO:teuthology.orchestra.run.smithi183.stdout:Updating Subscription Management repositories.
2024-04-10T22:40:16.982 INFO:teuthology.orchestra.run.smithi183.stdout:Unable to read consumer identity
2024-04-10T22:40:17.049 INFO:teuthology.orchestra.run.smithi183.stdout:
2024-04-10T22:40:17.049 INFO:teuthology.orchestra.run.smithi183.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register.
2024-04-10T22:40:17.049 INFO:teuthology.orchestra.run.smithi183.stdout:
2024-04-10T22:40:55.918 INFO:teuthology.orchestra.run.smithi183.stdout:Copr repo for python3-asyncssh owned by ceph    0.0  B/s |   0  B     00:38
2024-04-10T22:40:55.918 INFO:teuthology.orchestra.run.smithi183.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh':
2024-04-10T22:40:55.919 INFO:teuthology.orchestra.run.smithi183.stderr:  - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org]
2024-04-10T22:40:55.919 INFO:teuthology.orchestra.run.smithi183.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried
2024-04-10T22:40:56.068 INFO:teuthology.orchestra.run.smithi183.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh
2024-04-10T22:40:56.069 INFO:teuthology.orchestra.run.smithi183.stdout:Last metadata expiration check: 0:00:41 ago on Wed 10 Apr 2024 10:40:15 PM UTC.
2024-04-10T22:40:56.152 INFO:teuthology.orchestra.run.smithi183.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2024-04-10T22:40:56.156 INFO:teuthology.orchestra.run.smithi183.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available.
2024-04-10T22:40:56.199 INFO:teuthology.orchestra.run.smithi183.stderr:Error: No packages marked for reinstall.
2024-04-10T22:40:56.245 DEBUG:teuthology.orchestra.run.smithi183:> sudo yum reinstall -y kernel || true
2024-04-10T22:40:56.621 INFO:teuthology.orchestra.run.smithi183.stdout:Updating Subscription Management repositories.
2024-04-10T22:40:56.621 INFO:teuthology.orchestra.run.smithi183.stdout:Unable to read consumer identity
2024-04-10T22:40:56.678 INFO:teuthology.orchestra.run.smithi183.stdout:
2024-04-10T22:40:56.678 INFO:teuthology.orchestra.run.smithi183.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register.
2024-04-10T22:40:56.678 INFO:teuthology.orchestra.run.smithi183.stdout:
2024-04-10T22:41:36.343 INFO:teuthology.orchestra.run.smithi183.stdout:Copr repo for python3-asyncssh owned by ceph    0.0  B/s |   0  B     00:39
2024-04-10T22:41:36.343 INFO:teuthology.orchestra.run.smithi183.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh':
2024-04-10T22:41:36.344 INFO:teuthology.orchestra.run.smithi183.stderr:  - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org]
2024-04-10T22:41:36.344 INFO:teuthology.orchestra.run.smithi183.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried
2024-04-10T22:41:36.492 INFO:teuthology.orchestra.run.smithi183.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh
2024-04-10T22:41:36.493 INFO:teuthology.orchestra.run.smithi183.stdout:Last metadata expiration check: 0:01:21 ago on Wed 10 Apr 2024 10:40:15 PM UTC.
2024-04-10T22:41:36.560 INFO:teuthology.orchestra.run.smithi183.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available.
2024-04-10T22:41:36.561 INFO:teuthology.orchestra.run.smithi183.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available.
2024-04-10T22:41:36.597 INFO:teuthology.orchestra.run.smithi183.stderr:Error: No packages marked for reinstall.
2024-04-10T22:41:36.645 DEBUG:teuthology.orchestra.run.smithi183:> rpm -q kernel | sort -rV | head -n 1
2024-04-10T22:41:36.797 INFO:teuthology.orchestra.run.smithi183.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:41:36.797 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64
2024-04-10T22:41:36.797 INFO:teuthology.task.kernel:Newest distro kernel installed and running
2024-04-10T22:41:36.798 INFO:teuthology.task.kernel:Enabling kdb on mon.b...
2024-04-10T22:41:36.798 DEBUG:teuthology.orchestra.run.smithi183:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc
2024-04-10T22:41:36.840 INFO:teuthology.orchestra.run.smithi183.stdout:ttyS1
2024-04-10T22:41:36.853 DEBUG:teuthology.parallel:result is None
2024-04-10T22:41:36.854 INFO:teuthology.run_tasks:Running task internal.base...
2024-04-10T22:41:36.863 INFO:teuthology.task.internal:Creating test directory...
2024-04-10T22:41:36.863 DEBUG:teuthology.orchestra.run.smithi071:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2024-04-10T22:41:36.866 DEBUG:teuthology.orchestra.run.smithi183:> mkdir -p -m0755 -- /home/ubuntu/cephtest
2024-04-10T22:41:36.910 INFO:teuthology.run_tasks:Running task internal.archive_upload...
2024-04-10T22:41:36.949 INFO:teuthology.run_tasks:Running task internal.archive...
2024-04-10T22:41:36.996 INFO:teuthology.task.internal:Creating archive directory...
2024-04-10T22:41:36.996 DEBUG:teuthology.orchestra.run.smithi071:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2024-04-10T22:41:36.999 DEBUG:teuthology.orchestra.run.smithi183:> install -d -m0755 -- /home/ubuntu/cephtest/archive
2024-04-10T22:41:37.032 INFO:teuthology.run_tasks:Running task internal.coredump...
2024-04-10T22:41:37.049 INFO:teuthology.task.internal:Enabling coredump saving...
2024-04-10T22:41:37.049 DEBUG:teuthology.orchestra.run.smithi071:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2024-04-10T22:41:37.069 DEBUG:teuthology.orchestra.run.smithi183:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf
2024-04-10T22:41:37.105 INFO:teuthology.orchestra.run.smithi183.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-10T22:41:37.106 INFO:teuthology.orchestra.run.smithi071.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-10T22:41:37.115 INFO:teuthology.orchestra.run.smithi183.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-10T22:41:37.118 INFO:teuthology.orchestra.run.smithi071.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core
2024-04-10T22:41:37.120 INFO:teuthology.run_tasks:Running task internal.sudo...
2024-04-10T22:41:37.132 INFO:teuthology.task.internal:Configuring sudo...
2024-04-10T22:41:37.132 DEBUG:teuthology.orchestra.run.smithi071:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2024-04-10T22:41:37.162 DEBUG:teuthology.orchestra.run.smithi183:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers
2024-04-10T22:41:37.190 INFO:teuthology.run_tasks:Running task internal.syslog...
2024-04-10T22:41:37.204 INFO:teuthology.task.internal.syslog:Starting syslog monitoring...
2024-04-10T22:41:37.204 DEBUG:teuthology.orchestra.run.smithi071:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2024-04-10T22:41:37.233 DEBUG:teuthology.orchestra.run.smithi183:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog
2024-04-10T22:41:37.249 DEBUG:teuthology.orchestra.run.smithi071:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-10T22:41:37.335 DEBUG:teuthology.orchestra.run.smithi071:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-10T22:41:37.384 DEBUG:teuthology.orchestra.run.smithi071:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-10T22:41:37.469 DEBUG:teuthology.orchestra.run.smithi071:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-10T22:41:37.511 DEBUG:teuthology.orchestra.run.smithi071:> set -ex
2024-04-10T22:41:37.511 DEBUG:teuthology.orchestra.run.smithi071:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2024-04-10T22:41:37.577 DEBUG:teuthology.orchestra.run.smithi183:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-10T22:41:37.621 DEBUG:teuthology.orchestra.run.smithi183:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log
2024-04-10T22:41:37.683 DEBUG:teuthology.orchestra.run.smithi183:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-10T22:41:37.726 DEBUG:teuthology.orchestra.run.smithi183:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log
2024-04-10T22:41:37.767 DEBUG:teuthology.orchestra.run.smithi183:> set -ex
2024-04-10T22:41:37.767 DEBUG:teuthology.orchestra.run.smithi183:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf
2024-04-10T22:41:37.834 DEBUG:teuthology.orchestra.run.smithi071:> sudo service rsyslog restart
2024-04-10T22:41:37.836 DEBUG:teuthology.orchestra.run.smithi183:> sudo service rsyslog restart
2024-04-10T22:41:37.911 INFO:teuthology.orchestra.run.smithi071.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-10T22:41:37.925 INFO:teuthology.orchestra.run.smithi183.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-10T22:41:38.264 INFO:teuthology.run_tasks:Running task internal.timer...
2024-04-10T22:41:38.282 INFO:teuthology.task.internal:Starting timer...
2024-04-10T22:41:38.282 INFO:teuthology.run_tasks:Running task pcp...
2024-04-10T22:41:38.358 INFO:teuthology.run_tasks:Running task selinux...
2024-04-10T22:41:38.409 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}
2024-04-10T22:41:38.410 DEBUG:teuthology.orchestra.run.smithi071:> sudo service auditd rotate
2024-04-10T22:41:38.482 INFO:teuthology.orchestra.run.smithi071.stdout:Rotating logs:
2024-04-10T22:41:38.483 DEBUG:teuthology.orchestra.run.smithi183:> sudo service auditd rotate
2024-04-10T22:41:38.536 INFO:teuthology.orchestra.run.smithi183.stdout:Rotating logs:
2024-04-10T22:41:38.537 DEBUG:teuthology.task.selinux:Getting current SELinux state
2024-04-10T22:41:38.538 DEBUG:teuthology.orchestra.run.smithi071:> /usr/sbin/getenforce
2024-04-10T22:41:38.560 INFO:teuthology.orchestra.run.smithi071.stdout:Permissive
2024-04-10T22:41:38.561 DEBUG:teuthology.orchestra.run.smithi183:> /usr/sbin/getenforce
2024-04-10T22:41:38.594 INFO:teuthology.orchestra.run.smithi183.stdout:Permissive
2024-04-10T22:41:38.594 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi071.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi183.front.sepia.ceph.com': 'permissive'}
2024-04-10T22:41:38.594 DEBUG:teuthology.orchestra.run.smithi071:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2024-04-10T22:41:38.627 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:41:38.628 DEBUG:teuthology.orchestra.run.smithi183:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2024-04-10T22:41:38.662 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:41:38.662 INFO:teuthology.task.selinux:Putting SELinux into permissive mode
2024-04-10T22:41:38.662 DEBUG:teuthology.orchestra.run.smithi071:> sudo /usr/sbin/setenforce permissive
2024-04-10T22:41:38.697 DEBUG:teuthology.orchestra.run.smithi183:> sudo /usr/sbin/setenforce permissive
2024-04-10T22:41:38.730 INFO:teuthology.run_tasks:Running task ansible.cephlab...
2024-04-10T22:41:38.740 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main
2024-04-10T22:41:38.758 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}]
2024-04-10T22:41:38.760 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi071.front.sepia.ceph.com,smithi183.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml
2024-04-10T22:58:16.392 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650180/ansible_failures.yaml
2024-04-10T22:58:16.394 ERROR:teuthology.run_tasks:Saw exception from tasks.
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin
    super(CephLab, self).begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin
    self.execute_playbook()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook
    self._handle_failure(command, status)
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure
    raise AnsibleFailedError(failures)
teuthology.exceptions.AnsibleFailedError: ['<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at']
2024-04-10T22:58:16.622 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=05b616fc0dae4a80b5b8a3ba01c8a253
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin
    super(CephLab, self).begin()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin
    self.execute_playbook()
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook
    self._handle_failure(command, status)
  File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure
    raise AnsibleFailedError(failures)
teuthology.exceptions.AnsibleFailedError: ['<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at']
2024-04-10T22:58:16.624 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab
2024-04-10T22:58:16.636 INFO:teuthology.task.ansible:Skipping ansible cleanup...
2024-04-10T22:58:16.636 DEBUG:teuthology.run_tasks:Unwinding manager selinux
2024-04-10T22:58:16.683 DEBUG:teuthology.orchestra.run.smithi071:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log
2024-04-10T22:58:16.763 DEBUG:teuthology.orchestra.run.smithi183:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log
2024-04-10T22:58:16.828 DEBUG:teuthology.orchestra.run.smithi071:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2024-04-10T22:58:16.859 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:58:16.860 DEBUG:teuthology.orchestra.run.smithi183:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0
2024-04-10T22:58:16.890 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:58:16.891 DEBUG:teuthology.run_tasks:Unwinding manager pcp
2024-04-10T22:58:16.900 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer
2024-04-10T22:58:16.960 INFO:teuthology.task.internal:Duration was 998.678219 seconds
2024-04-10T22:58:16.961 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog
2024-04-10T22:58:17.009 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring...
2024-04-10T22:58:17.009 DEBUG:teuthology.orchestra.run.smithi071:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart
2024-04-10T22:58:17.011 DEBUG:teuthology.orchestra.run.smithi183:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart
2024-04-10T22:58:17.059 INFO:teuthology.orchestra.run.smithi071.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-10T22:58:17.060 INFO:teuthology.orchestra.run.smithi183.stderr:Redirecting to /bin/systemctl restart rsyslog.service
2024-04-10T22:58:17.514 INFO:teuthology.task.internal.syslog:Checking logs for errors...
2024-04-10T22:58:17.515 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi071.front.sepia.ceph.com
2024-04-10T22:58:17.515 DEBUG:teuthology.orchestra.run.smithi071:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1
2024-04-10T22:58:17.539 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi183.front.sepia.ceph.com
2024-04-10T22:58:17.539 DEBUG:teuthology.orchestra.run.smithi183:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1
2024-04-10T22:58:17.562 INFO:teuthology.task.internal.syslog:Compressing syslogs...
2024-04-10T22:58:17.562 DEBUG:teuthology.orchestra.run.smithi071:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
2024-04-10T22:58:17.581 DEBUG:teuthology.orchestra.run.smithi183:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
2024-04-10T22:58:17.645 INFO:teuthology.task.internal.syslog:Gathering journactl -b0...
2024-04-10T22:58:17.645 DEBUG:teuthology.orchestra.run.smithi071:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz
2024-04-10T22:58:17.666 DEBUG:teuthology.orchestra.run.smithi183:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz
2024-04-10T22:58:17.753 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo
2024-04-10T22:58:17.762 INFO:teuthology.task.internal:Restoring /etc/sudoers...
2024-04-10T22:58:17.762 DEBUG:teuthology.orchestra.run.smithi071:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers
2024-04-10T22:58:17.790 DEBUG:teuthology.orchestra.run.smithi183:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers
2024-04-10T22:58:17.816 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
2024-04-10T22:58:17.825 DEBUG:teuthology.orchestra.run.smithi071:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
2024-04-10T22:58:17.832 DEBUG:teuthology.orchestra.run.smithi183:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
2024-04-10T22:58:17.856 INFO:teuthology.orchestra.run.smithi071.stdout:kernel.core_pattern = core
2024-04-10T22:58:17.882 INFO:teuthology.orchestra.run.smithi183.stdout:kernel.core_pattern = core
2024-04-10T22:58:17.905 DEBUG:teuthology.orchestra.run.smithi071:> test -e /home/ubuntu/cephtest/archive/coredump
2024-04-10T22:58:17.939 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:58:17.940 DEBUG:teuthology.orchestra.run.smithi183:> test -e /home/ubuntu/cephtest/archive/coredump
2024-04-10T22:58:17.961 DEBUG:teuthology.orchestra.run:got remote process result: 1
2024-04-10T22:58:17.962 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive
2024-04-10T22:58:17.971 INFO:teuthology.task.internal:Transferring archived files...
2024-04-10T22:58:17.973 DEBUG:teuthology.misc:Transferring archived files from smithi071:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650180/remote/smithi071
2024-04-10T22:58:17.974 DEBUG:teuthology.orchestra.run.smithi071:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- .
2024-04-10T22:58:18.027 DEBUG:teuthology.misc:Transferring archived files from smithi183:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650180/remote/smithi183
2024-04-10T22:58:18.028 DEBUG:teuthology.orchestra.run.smithi183:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- .
2024-04-10T22:58:18.082 INFO:teuthology.task.internal:Removing archive directory...
2024-04-10T22:58:18.082 DEBUG:teuthology.orchestra.run.smithi071:> rm -rf -- /home/ubuntu/cephtest/archive
2024-04-10T22:58:18.084 DEBUG:teuthology.orchestra.run.smithi183:> rm -rf -- /home/ubuntu/cephtest/archive
2024-04-10T22:58:18.132 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload
2024-04-10T22:58:18.143 INFO:teuthology.task.internal:Not uploading archives.
2024-04-10T22:58:18.144 DEBUG:teuthology.run_tasks:Unwinding manager internal.base
2024-04-10T22:58:18.168 INFO:teuthology.task.internal:Tidying up after the test...
2024-04-10T22:58:18.168 DEBUG:teuthology.orchestra.run.smithi071:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest
2024-04-10T22:58:18.171 DEBUG:teuthology.orchestra.run.smithi183:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest
2024-04-10T22:58:18.185 INFO:teuthology.orchestra.run.smithi071.stdout:   262171      4 drwxr-xr-x   2  ubuntu   ubuntu       4096 Apr 10 22:58 /home/ubuntu/cephtest
2024-04-10T22:58:18.189 INFO:teuthology.orchestra.run.smithi183.stdout:   262171      4 drwxr-xr-x   2  ubuntu   ubuntu       4096 Apr 10 22:58 /home/ubuntu/cephtest
2024-04-10T22:58:18.190 DEBUG:teuthology.run_tasks:Unwinding manager kernel
2024-04-10T22:58:18.217 DEBUG:teuthology.run_tasks:Unwinding manager console_log
2024-04-10T22:58:18.318 INFO:teuthology.nuke:Checking targets against current locks
2024-04-10T22:58:18.354 DEBUG:teuthology.nuke:shortname: smithi071
2024-04-10T22:58:18.354 INFO:teuthology.task.internal.check_lock:Checking locks...
2024-04-10T22:58:18.398 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi071.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650180', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:15.479212', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMsKWubXofcVlZXZdH0ffoh6ztUo2wLmmZF1dA/7QzvD7Z+aFib5ONAt+U28A6pXyl8EDWj3rCv/PX/D7OvRf5s='}
2024-04-10T22:58:18.412 DEBUG:teuthology.nuke:shortname: smithi183
2024-04-10T22:58:18.412 INFO:teuthology.task.internal.check_lock:Checking locks...
2024-04-10T22:58:18.444 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi183.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650180', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:15.480655', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC+OQVzPG1Sf1eQr1uHKpf6a/BJXRAPQQRrpAgNFKmwthQhRYAo2vZ2P1tsw5aUj6NvbWxHZ8xeID9yLtf+lw7g='}
2024-04-10T22:58:18.515 INFO:teuthology.orchestra.console.smithi071:Power off
2024-04-10T22:58:18.515 DEBUG:teuthology.orchestra.console.smithi071:pexpect command: ipmitool -H smithi071.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2024-04-10T22:58:18.550 INFO:teuthology.orchestra.console.smithi183:Power off
2024-04-10T22:58:18.550 DEBUG:teuthology.orchestra.console.smithi183:pexpect command: ipmitool -H smithi183.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off
2024-04-10T22:58:18.567 DEBUG:teuthology.orchestra.console.smithi071:power off output: Chassis Power Control: Down/Off
2024-04-10T22:58:18.568 DEBUG:teuthology.orchestra.console.smithi071:pexpect command: ipmitool -H smithi071.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-10T22:58:18.605 DEBUG:teuthology.orchestra.console.smithi183:power off output: Chassis Power Control: Down/Off
2024-04-10T22:58:18.605 DEBUG:teuthology.orchestra.console.smithi183:pexpect command: ipmitool -H smithi183.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-10T22:58:18.620 DEBUG:teuthology.orchestra.console.smithi071:check power output: Chassis Power is on
2024-04-10T22:58:18.659 DEBUG:teuthology.orchestra.console.smithi183:check power output: Chassis Power is on
2024-04-10T22:58:22.622 DEBUG:teuthology.orchestra.console.smithi071:pexpect command: ipmitool -H smithi071.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-10T22:58:22.660 DEBUG:teuthology.orchestra.console.smithi183:pexpect command: ipmitool -H smithi183.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-10T22:58:22.736 DEBUG:teuthology.orchestra.console.smithi071:check power output: Chassis Power is on
2024-04-10T22:58:22.771 DEBUG:teuthology.orchestra.console.smithi183:check power output: Chassis Power is on
2024-04-10T22:58:26.738 DEBUG:teuthology.orchestra.console.smithi071:pexpect command: ipmitool -H smithi071.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-10T22:58:26.771 DEBUG:teuthology.orchestra.console.smithi183:pexpect command: ipmitool -H smithi183.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status
2024-04-10T22:58:26.850 DEBUG:teuthology.orchestra.console.smithi071:check power output: Chassis Power is off
2024-04-10T22:58:26.883 DEBUG:teuthology.orchestra.console.smithi183:check power output: Chassis Power is off
2024-04-10T22:58:26.952 INFO:teuthology.orchestra.console.smithi071:Power off completed
2024-04-10T22:58:26.985 INFO:teuthology.orchestra.console.smithi183:Power off completed
2024-04-10T22:58:27.091 INFO:teuthology.run:Summary data:
description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0
  0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls
  rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench
  mon_election/classic}
duration: 998.6782190799713
failure_reason: '[''<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,'',
  ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling
  a Copr repository. Please note that this repository is not part'', ''Error: Failed
  to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64:
  Network is unreachable'', ''Please do not file bug reports about these packages
  in Fedora'', ''The Fedora Project does not exercise any power over the contents
  of'', ''and packages are not held to any quality or security level.'', ''of the
  main distribution, and quality may vary.'', ''this repository beyond the rules outlined
  in the Copr FAQ at'']'
owner: scheduled_teuthology@teuthology
sentry_event: https://sentry.ceph.com/organizations/ceph/?query=05b616fc0dae4a80b5b8a3ba01c8a253
status: dead
success: false

2024-04-10T22:58:27.091 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/
2024-04-10T22:58:27.191 INFO:teuthology.run:DEAD