2022-05-05T14:11:04.073 INFO:root:teuthology version: 1.1.1.dev331+gfd5c3166 2022-05-05T14:11:04.074 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_yuriw@teuthology --archive /home/teuthworker/archive/yuriw-2022-05-05_14:00:38-rados-pacific-distro-default-smithi/6823083 --verbose --name yuriw-2022-05-05_14:00:38-rados-pacific-distro-default-smithi --description rados/cephadm/mgr-nfs-upgrade/{0-distro/centos_8.stream_container_tools 1-bootstrap/16.2.4 1-start 2-nfs 3-upgrade-with-workload 4-final} -- /home/teuthworker/archive/yuriw-2022-05-05_14:00:38-rados-pacific-distro-default-smithi/6823083/orig.config.yaml 2022-05-05T14:11:04.114 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/ 2022-05-05T14:11:04.176 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/yuriw-2022-05-05_14:00:38-rados-pacific-distro-default-smithi/6823083 branch: pacific description: rados/cephadm/mgr-nfs-upgrade/{0-distro/centos_8.stream_container_tools 1-bootstrap/16.2.4 1-start 2-nfs 3-upgrade-with-workload 4-final} email: null first_in_suite: false job_id: '6823083' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: yuriw-2022-05-05_14:00:38-rados-pacific-distro-default-smithi nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: pacific ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-whitelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 install: ceph: flavor: default sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: pacific sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d owner: scheduled_yuriw@teuthology priority: 71 repo: https://github.com/ceph/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d sleep_before_teardown: 0 suite: rados suite_branch: pacific suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_4fa079ba14503defa8dc257d7c2d506ebefcfe6d/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d targets: smithi022.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDi7nlJhOlarjgd+HtS+N1P2pGWAMLqtZ/s6QnQfFgWu8Rq7LJT4zhBit2o1C8VgdFGmhVj+CWrlRt5OrKGVc8IzwLJU+VQ2zMlAgyg9pDAD9Fca51/h6KV6fTQYL59xGhcHSAUNxA5vGqoavgl8sfMXiMEq/V8uLYkTvfLs9gHZsJdBYci0vFsKKDmJuX5XfQuH6TU/iBRyt2Gbm3ijbU+QxWCz2BOYCBj3QgvUy8SKIXi0Z5PoxsnaOIYRVOOxWdPXmSNOIW9uOETn8YZfSPSQW4im3dj7eJc3m8tqcySBbwKDAYhdK86vQbcZdWc20ihG9y0f7+NrDt6Ljbemu1PWF7pv1UpKNZrtu57GspfRTpGZFZNiJRBIGtEpkJkxXNEdlrIq0vndYfQu4PmW0dvr11LL+jvbCCMFu3bD2Sw0fJZgoZKlC7eMTS0FyLlHf2ljHEkj/tZ+JF3ATu7K6/uLLiFQk+vfrAKylkNr7pamznjXmhX+wxMpiU81Wvn3s= smithi167.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDFQqmrIfXQvWA8gig8O5UWa4y6kulww1gIWPkHnUPtrkNbqgISAJtWKyLcUGI/9FU/aI+lUwZ4h2ukAKKECHh0rCWd5NiJpGOaig8G7EPJcXMgrlT2XTjuhUPVpudaLOi+Z0ZvafpXUx2pz5M3RDI81hJnXh4/KOq6TU2ATQtd54zfoMZgsVROlPgUsWUTJFi5tMbQ1h0SUnzgxRWBHMCUHb/D5QgPe0pEbxUE+NkgG1ClW/Bxi/U29n+3IjiYd1W3283aNDXi+m6zNiLohSsqbOmjiRtM9+jo1VuL1m+RQkiJOD00xKOsc5XFFz1FkQYroY7uQHYUd14p9F4KpPkyq6+APN6igMPBLBzXkwxhr9BtxzI59hDv1OYh8XKW+Qv2eM4jv+oeRoRzqI3ZzLrAk7NRPpeoG21FqXiZXHkvsJoDzSvfZPCccqz5ZQgyETKt1lkiN6rSDUaptHQnyVf9qIyhDWt7o3hh2HtHsQwVA2xL7RmwY5pXLNqAEApVFtE= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - cephadm: avoid_pacific_features: true cephadm_branch: v16.2.4 cephadm_git_url: https://github.com/ceph/ceph image: docker.io/ceph/ceph:v16.2.4 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - ceph config get mgr mgr/cephadm/migration_current | grep 5 teuthology_branch: master teuthology_sha1: fd5c31669e51177d3c57bc0420279c5d1e944497 timestamp: 2022-05-05_14:00:38 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: yuriw verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.456 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2022-05-05T14:11:04.177 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_4fa079ba14503defa8dc257d7c2d506ebefcfe6d/qa; will attempt to use it 2022-05-05T14:11:04.178 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_4fa079ba14503defa8dc257d7c2d506ebefcfe6d/qa/tasks 2022-05-05T14:11:04.178 INFO:teuthology.run_tasks:Running task internal.check_packages... 2022-05-05T14:11:04.180 INFO:teuthology.task.internal:Checking packages... 2022-05-05T14:11:04.200 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '4fa079ba14503defa8dc257d7c2d506ebefcfe6d' 2022-05-05T14:11:04.201 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-05-05T14:11:04.201 INFO:teuthology.packaging:ref: None 2022-05-05T14:11:04.202 INFO:teuthology.packaging:tag: None 2022-05-05T14:11:04.202 INFO:teuthology.packaging:branch: pacific 2022-05-05T14:11:04.202 INFO:teuthology.packaging:sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d 2022-05-05T14:11:04.203 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2022-05-05T14:11:04.585 INFO:teuthology.task.internal:Found packages for ceph version 16.2.7-981.g73636a1b 2022-05-05T14:11:04.585 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2022-05-05T14:11:04.594 INFO:teuthology.task.internal:no buildpackages task found 2022-05-05T14:11:04.594 INFO:teuthology.run_tasks:Running task internal.save_config... 2022-05-05T14:11:04.609 INFO:teuthology.task.internal:Saving configuration 2022-05-05T14:11:04.624 INFO:teuthology.run_tasks:Running task internal.check_lock... 2022-05-05T14:11:04.636 INFO:teuthology.task.internal.check_lock:Checking locks... 2022-05-05T14:11:04.657 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi022.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2022-05-05_14:00:38-rados-pacific-distro-default-smithi/6823083', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2022-05-05 14:06:06.911442', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBN+r//6LcBOV9sIFa8GfFJNmXNNc/9PtEXZBUcQtWNlrmMZ7G5nWzCIG9Xcil+EnEPlxaI61s4o1VQHE7QtDBhM='} 2022-05-05T14:11:04.677 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi167.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2022-05-05_14:00:38-rados-pacific-distro-default-smithi/6823083', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2022-05-05 14:06:06.910133', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBONC04kqvQJMHEKMEWx2o8OOKQR30mbuQU9UfhKLpACIUSJsPY8FcZa0foa+9cG60vKubkF7JFXuXvfZw6Uic5A='} 2022-05-05T14:11:04.677 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2022-05-05T14:11:04.687 INFO:teuthology.task.internal:roles: ubuntu@smithi022.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2022-05-05T14:11:04.688 INFO:teuthology.task.internal:roles: ubuntu@smithi167.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2022-05-05T14:11:04.688 INFO:teuthology.run_tasks:Running task console_log... 2022-05-05T14:11:04.765 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7ffbbe3edd90>, signals=[15]) 2022-05-05T14:11:04.766 INFO:teuthology.run_tasks:Running task internal.connect... 2022-05-05T14:11:04.774 INFO:teuthology.task.internal:Opening connections... 2022-05-05T14:11:04.775 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi022.front.sepia.ceph.com 2022-05-05T14:11:04.776 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-05-05T14:11:04.860 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi167.front.sepia.ceph.com 2022-05-05T14:11:04.861 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-05-05T14:11:04.938 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2022-05-05T14:11:04.947 DEBUG:teuthology.orchestra.run.smithi022:> uname -m 2022-05-05T14:11:04.969 INFO:teuthology.orchestra.run.smithi022.stdout:x86_64 2022-05-05T14:11:04.969 DEBUG:teuthology.orchestra.run.smithi022:> cat /etc/os-release 2022-05-05T14:11:05.030 INFO:teuthology.orchestra.run.smithi022.stdout:NAME="CentOS Stream" 2022-05-05T14:11:05.030 INFO:teuthology.orchestra.run.smithi022.stdout:VERSION="8" 2022-05-05T14:11:05.030 INFO:teuthology.orchestra.run.smithi022.stdout:ID="centos" 2022-05-05T14:11:05.030 INFO:teuthology.orchestra.run.smithi022.stdout:ID_LIKE="rhel fedora" 2022-05-05T14:11:05.031 INFO:teuthology.orchestra.run.smithi022.stdout:VERSION_ID="8" 2022-05-05T14:11:05.031 INFO:teuthology.orchestra.run.smithi022.stdout:PLATFORM_ID="platform:el8" 2022-05-05T14:11:05.031 INFO:teuthology.orchestra.run.smithi022.stdout:PRETTY_NAME="CentOS Stream 8" 2022-05-05T14:11:05.031 INFO:teuthology.orchestra.run.smithi022.stdout:ANSI_COLOR="0;31" 2022-05-05T14:11:05.031 INFO:teuthology.orchestra.run.smithi022.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2022-05-05T14:11:05.032 INFO:teuthology.orchestra.run.smithi022.stdout:HOME_URL="https://centos.org/" 2022-05-05T14:11:05.032 INFO:teuthology.orchestra.run.smithi022.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2022-05-05T14:11:05.032 INFO:teuthology.orchestra.run.smithi022.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2022-05-05T14:11:05.032 INFO:teuthology.orchestra.run.smithi022.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2022-05-05T14:11:05.033 INFO:teuthology.lock.ops:Updating smithi022.front.sepia.ceph.com on lock server 2022-05-05T14:11:05.052 DEBUG:teuthology.orchestra.run.smithi167:> uname -m 2022-05-05T14:11:05.072 INFO:teuthology.orchestra.run.smithi167.stdout:x86_64 2022-05-05T14:11:05.072 DEBUG:teuthology.orchestra.run.smithi167:> cat /etc/os-release 2022-05-05T14:11:05.130 INFO:teuthology.orchestra.run.smithi167.stdout:NAME="CentOS Stream" 2022-05-05T14:11:05.131 INFO:teuthology.orchestra.run.smithi167.stdout:VERSION="8" 2022-05-05T14:11:05.131 INFO:teuthology.orchestra.run.smithi167.stdout:ID="centos" 2022-05-05T14:11:05.131 INFO:teuthology.orchestra.run.smithi167.stdout:ID_LIKE="rhel fedora" 2022-05-05T14:11:05.131 INFO:teuthology.orchestra.run.smithi167.stdout:VERSION_ID="8" 2022-05-05T14:11:05.132 INFO:teuthology.orchestra.run.smithi167.stdout:PLATFORM_ID="platform:el8" 2022-05-05T14:11:05.132 INFO:teuthology.orchestra.run.smithi167.stdout:PRETTY_NAME="CentOS Stream 8" 2022-05-05T14:11:05.132 INFO:teuthology.orchestra.run.smithi167.stdout:ANSI_COLOR="0;31" 2022-05-05T14:11:05.132 INFO:teuthology.orchestra.run.smithi167.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2022-05-05T14:11:05.133 INFO:teuthology.orchestra.run.smithi167.stdout:HOME_URL="https://centos.org/" 2022-05-05T14:11:05.133 INFO:teuthology.orchestra.run.smithi167.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2022-05-05T14:11:05.133 INFO:teuthology.orchestra.run.smithi167.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2022-05-05T14:11:05.133 INFO:teuthology.orchestra.run.smithi167.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2022-05-05T14:11:05.134 INFO:teuthology.lock.ops:Updating smithi167.front.sepia.ceph.com on lock server 2022-05-05T14:11:05.151 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2022-05-05T14:11:05.162 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2022-05-05T14:11:05.284 INFO:teuthology.task.internal:Checking for old test directory... 2022-05-05T14:11:05.284 DEBUG:teuthology.orchestra.run.smithi022:> test '!' -e /home/ubuntu/cephtest 2022-05-05T14:11:05.287 DEBUG:teuthology.orchestra.run.smithi167:> test '!' -e /home/ubuntu/cephtest 2022-05-05T14:11:05.306 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2022-05-05T14:11:05.445 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2022-05-05T14:11:05.446 DEBUG:teuthology.orchestra.run.smithi022:> test -z $(ls -A /var/lib/ceph) 2022-05-05T14:11:05.448 DEBUG:teuthology.orchestra.run.smithi167:> test -z $(ls -A /var/lib/ceph) 2022-05-05T14:11:05.474 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2022-05-05T14:11:05.644 INFO:teuthology.run_tasks:Running task kernel... 2022-05-05T14:11:05.659 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2022-05-05T14:11:05.660 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}, 'host.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2022-05-05T14:11:05.660 DEBUG:teuthology.orchestra.run.smithi022:> test -f /run/.containerenv -o -f /.dockerenv 2022-05-05T14:11:05.661 DEBUG:teuthology.orchestra.run.smithi167:> test -f /run/.containerenv -o -f /.dockerenv 2022-05-05T14:11:05.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-05-05T14:11:05.681 DEBUG:teuthology.orchestra.run.smithi167:> uname -r 2022-05-05T14:11:05.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-05-05T14:11:05.683 DEBUG:teuthology.orchestra.run.smithi022:> uname -r 2022-05-05T14:11:05.740 INFO:teuthology.orchestra.run.smithi167.stdout:4.18.0-383.el8.x86_64 2022-05-05T14:11:05.740 INFO:teuthology.task.kernel:Running kernel on smithi167: 4.18.0-383.el8.x86_64 2022-05-05T14:11:05.741 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum install -y kernel 2022-05-05T14:11:05.742 INFO:teuthology.orchestra.run.smithi022.stdout:4.18.0-383.el8.x86_64 2022-05-05T14:11:05.743 INFO:teuthology.task.kernel:Running kernel on smithi022: 4.18.0-383.el8.x86_64 2022-05-05T14:11:05.743 DEBUG:teuthology.orchestra.run.smithi022:> sudo yum install -y kernel 2022-05-05T14:11:08.489 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS-8 - AppStream 30 MB/s | 22 MB 00:00 2022-05-05T14:11:08.560 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS-8 - AppStream 27 MB/s | 22 MB 00:00 2022-05-05T14:11:12.876 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS-8 - Base 41 MB/s | 22 MB 00:00 2022-05-05T14:11:13.322 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS-8 - Base 28 MB/s | 22 MB 00:00 2022-05-05T14:11:16.159 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS-8 - Extras 159 kB/s | 18 kB 00:00 2022-05-05T14:11:16.536 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS-8 - PowerTools 18 MB/s | 4.6 MB 00:00 2022-05-05T14:11:16.863 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS-8 - Extras 188 kB/s | 18 kB 00:00 2022-05-05T14:11:17.248 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS-8 - PowerTools 18 MB/s | 4.6 MB 00:00 2022-05-05T14:11:17.526 INFO:teuthology.orchestra.run.smithi167.stdout:Copr repo for python3-asyncssh owned by ceph 30 kB/s | 3.5 kB 00:00 2022-05-05T14:11:18.306 INFO:teuthology.orchestra.run.smithi022.stdout:Copr repo for python3-asyncssh owned by ceph 28 kB/s | 3.5 kB 00:00 2022-05-05T14:11:18.735 INFO:teuthology.orchestra.run.smithi167.stdout:Extra Packages for Enterprise Linux 10 MB/s | 11 MB 00:01 2022-05-05T14:11:19.548 INFO:teuthology.orchestra.run.smithi022.stdout:Extra Packages for Enterprise Linux 9.9 MB/s | 11 MB 00:01 2022-05-05T14:11:21.502 INFO:teuthology.orchestra.run.smithi167.stdout:lab-extras 446 kB/s | 24 kB 00:00 2022-05-05T14:11:22.506 INFO:teuthology.orchestra.run.smithi022.stdout:lab-extras 423 kB/s | 24 kB 00:00 2022-05-05T14:11:24.033 INFO:teuthology.orchestra.run.smithi167.stdout:Package kernel-4.18.0-383.el8.x86_64 is already installed. 2022-05-05T14:11:24.033 INFO:teuthology.orchestra.run.smithi167.stdout:Package kernel-4.18.0-305.3.1.el8.x86_64 is already installed. 2022-05-05T14:11:24.168 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2022-05-05T14:11:24.170 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2022-05-05T14:11:24.170 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2022-05-05T14:11:24.301 DEBUG:teuthology.orchestra.run.smithi167:> echo no | sudo yum reinstall kernel || true 2022-05-05T14:11:24.894 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:03 ago on Thu 05 May 2022 02:11:21 PM UTC. 2022-05-05T14:11:25.278 INFO:teuthology.orchestra.run.smithi022.stdout:Package kernel-4.18.0-383.el8.x86_64 is already installed. 2022-05-05T14:11:25.279 INFO:teuthology.orchestra.run.smithi022.stdout:Package kernel-4.18.0-305.3.1.el8.x86_64 is already installed. 2022-05-05T14:11:25.408 INFO:teuthology.orchestra.run.smithi022.stdout:Dependencies resolved. 2022-05-05T14:11:25.409 INFO:teuthology.orchestra.run.smithi022.stdout:Nothing to do. 2022-05-05T14:11:25.409 INFO:teuthology.orchestra.run.smithi022.stdout:Complete! 2022-05-05T14:11:25.528 DEBUG:teuthology.orchestra.run.smithi022:> echo no | sudo yum reinstall kernel || true 2022-05-05T14:11:25.731 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2022-05-05T14:11:25.732 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2022-05-05T14:11:25.732 INFO:teuthology.orchestra.run.smithi167.stdout: Package Architecture Version Repository Size 2022-05-05T14:11:25.732 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2022-05-05T14:11:25.733 INFO:teuthology.orchestra.run.smithi167.stdout:Reinstalling: 2022-05-05T14:11:25.733 INFO:teuthology.orchestra.run.smithi167.stdout: kernel x86_64 4.18.0-383.el8 CentOS-Base 8.1 M 2022-05-05T14:11:25.733 INFO:teuthology.orchestra.run.smithi167.stdout: 2022-05-05T14:11:25.733 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2022-05-05T14:11:25.733 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2022-05-05T14:11:25.734 INFO:teuthology.orchestra.run.smithi167.stdout: 2022-05-05T14:11:25.734 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 8.1 M 2022-05-05T14:11:25.734 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 0 2022-05-05T14:11:25.734 INFO:teuthology.orchestra.run.smithi167.stderr:Operation aborted. 2022-05-05T14:11:25.807 INFO:teuthology.orchestra.run.smithi167.stdout:Is this ok [y/N]: 2022-05-05T14:11:25.808 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum reinstall -y kernel || true 2022-05-05T14:11:26.107 INFO:teuthology.orchestra.run.smithi022.stdout:Last metadata expiration check: 0:00:04 ago on Thu 05 May 2022 02:11:22 PM UTC. 2022-05-05T14:11:26.356 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:05 ago on Thu 05 May 2022 02:11:21 PM UTC. 2022-05-05T14:11:27.013 INFO:teuthology.orchestra.run.smithi022.stdout:Dependencies resolved. 2022-05-05T14:11:27.014 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2022-05-05T14:11:27.015 INFO:teuthology.orchestra.run.smithi022.stdout: Package Architecture Version Repository Size 2022-05-05T14:11:27.015 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2022-05-05T14:11:27.015 INFO:teuthology.orchestra.run.smithi022.stdout:Reinstalling: 2022-05-05T14:11:27.016 INFO:teuthology.orchestra.run.smithi022.stdout: kernel x86_64 4.18.0-383.el8 CentOS-Base 8.1 M 2022-05-05T14:11:27.016 INFO:teuthology.orchestra.run.smithi022.stdout: 2022-05-05T14:11:27.016 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction Summary 2022-05-05T14:11:27.016 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2022-05-05T14:11:27.017 INFO:teuthology.orchestra.run.smithi022.stdout: 2022-05-05T14:11:27.017 INFO:teuthology.orchestra.run.smithi022.stdout:Total download size: 8.1 M 2022-05-05T14:11:27.017 INFO:teuthology.orchestra.run.smithi022.stdout:Installed size: 0 2022-05-05T14:11:27.018 INFO:teuthology.orchestra.run.smithi022.stderr:Operation aborted. 2022-05-05T14:11:27.095 INFO:teuthology.orchestra.run.smithi022.stdout:Is this ok [y/N]: 2022-05-05T14:11:27.095 DEBUG:teuthology.orchestra.run.smithi022:> sudo yum reinstall -y kernel || true 2022-05-05T14:11:27.195 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2022-05-05T14:11:27.196 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2022-05-05T14:11:27.197 INFO:teuthology.orchestra.run.smithi167.stdout: Package Architecture Version Repository Size 2022-05-05T14:11:27.197 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2022-05-05T14:11:27.197 INFO:teuthology.orchestra.run.smithi167.stdout:Reinstalling: 2022-05-05T14:11:27.197 INFO:teuthology.orchestra.run.smithi167.stdout: kernel x86_64 4.18.0-383.el8 CentOS-Base 8.1 M 2022-05-05T14:11:27.198 INFO:teuthology.orchestra.run.smithi167.stdout: 2022-05-05T14:11:27.198 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2022-05-05T14:11:27.198 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2022-05-05T14:11:27.198 INFO:teuthology.orchestra.run.smithi167.stdout: 2022-05-05T14:11:27.198 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 8.1 M 2022-05-05T14:11:27.199 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 0 2022-05-05T14:11:27.199 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2022-05-05T14:11:27.437 INFO:teuthology.orchestra.run.smithi167.stdout:kernel-4.18.0-383.el8.x86_64.rpm 34 MB/s | 8.1 MB 00:00 2022-05-05T14:11:27.438 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2022-05-05T14:11:27.439 INFO:teuthology.orchestra.run.smithi167.stdout:Total 33 MB/s | 8.1 MB 00:00 2022-05-05T14:11:27.525 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2022-05-05T14:11:27.668 INFO:teuthology.orchestra.run.smithi022.stdout:Last metadata expiration check: 0:00:05 ago on Thu 05 May 2022 02:11:22 PM UTC. 2022-05-05T14:11:27.930 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2022-05-05T14:11:27.930 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2022-05-05T14:11:28.092 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2022-05-05T14:11:28.097 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2022-05-05T14:11:28.390 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2022-05-05T14:11:28.587 INFO:teuthology.orchestra.run.smithi167.stdout: Reinstalling : kernel-4.18.0-383.el8.x86_64 1/2 2022-05-05T14:11:28.587 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : kernel-4.18.0-383.el8.x86_64 2/2 2022-05-05T14:11:28.610 INFO:teuthology.orchestra.run.smithi022.stdout:Dependencies resolved. 2022-05-05T14:11:28.611 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2022-05-05T14:11:28.611 INFO:teuthology.orchestra.run.smithi022.stdout: Package Architecture Version Repository Size 2022-05-05T14:11:28.612 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2022-05-05T14:11:28.612 INFO:teuthology.orchestra.run.smithi022.stdout:Reinstalling: 2022-05-05T14:11:28.612 INFO:teuthology.orchestra.run.smithi022.stdout: kernel x86_64 4.18.0-383.el8 CentOS-Base 8.1 M 2022-05-05T14:11:28.612 INFO:teuthology.orchestra.run.smithi022.stdout: 2022-05-05T14:11:28.613 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction Summary 2022-05-05T14:11:28.613 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2022-05-05T14:11:28.613 INFO:teuthology.orchestra.run.smithi022.stdout: 2022-05-05T14:11:28.613 INFO:teuthology.orchestra.run.smithi022.stdout:Total download size: 8.1 M 2022-05-05T14:11:28.614 INFO:teuthology.orchestra.run.smithi022.stdout:Installed size: 0 2022-05-05T14:11:28.614 INFO:teuthology.orchestra.run.smithi022.stdout:Downloading Packages: 2022-05-05T14:11:28.875 INFO:teuthology.orchestra.run.smithi022.stdout:kernel-4.18.0-383.el8.x86_64.rpm 31 MB/s | 8.1 MB 00:00 2022-05-05T14:11:28.876 INFO:teuthology.orchestra.run.smithi022.stdout:-------------------------------------------------------------------------------- 2022-05-05T14:11:28.876 INFO:teuthology.orchestra.run.smithi022.stdout:Total 31 MB/s | 8.1 MB 00:00 2022-05-05T14:11:28.970 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction check 2022-05-05T14:11:28.974 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: kernel-4.18.0-383.el8.x86_64 2/2 2022-05-05T14:11:28.974 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : kernel-4.18.0-383.el8.x86_64 1/2 2022-05-05T14:11:29.359 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction check succeeded. 2022-05-05T14:11:29.359 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction test 2022-05-05T14:11:29.465 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : kernel-4.18.0-383.el8.x86_64 2/2 2022-05-05T14:11:29.466 INFO:teuthology.orchestra.run.smithi167.stdout: 2022-05-05T14:11:29.466 INFO:teuthology.orchestra.run.smithi167.stdout:Reinstalled: 2022-05-05T14:11:29.466 INFO:teuthology.orchestra.run.smithi167.stdout: kernel-4.18.0-383.el8.x86_64 2022-05-05T14:11:29.467 INFO:teuthology.orchestra.run.smithi167.stdout: 2022-05-05T14:11:29.467 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2022-05-05T14:11:29.508 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction test succeeded. 2022-05-05T14:11:29.517 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction 2022-05-05T14:11:29.552 DEBUG:teuthology.orchestra.run.smithi167:> rpm -q kernel --last | head -n 1 2022-05-05T14:11:29.673 INFO:teuthology.orchestra.run.smithi167.stdout:kernel-4.18.0-383.el8.x86_64 Thu 05 May 2022 02:11:28 PM UTC 2022-05-05T14:11:29.674 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-383.el8.x86_64 2022-05-05T14:11:29.675 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2022-05-05T14:11:29.675 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2022-05-05T14:11:29.675 DEBUG:teuthology.orchestra.run.smithi167:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2022-05-05T14:11:29.712 INFO:teuthology.orchestra.run.smithi167.stdout:ttyS1 2022-05-05T14:11:29.725 DEBUG:teuthology.parallel:result is None 2022-05-05T14:11:29.818 INFO:teuthology.orchestra.run.smithi022.stdout: Preparing : 1/1 2022-05-05T14:11:30.011 INFO:teuthology.orchestra.run.smithi022.stdout: Reinstalling : kernel-4.18.0-383.el8.x86_64 1/2 2022-05-05T14:11:30.012 INFO:teuthology.orchestra.run.smithi022.stdout: Cleanup : kernel-4.18.0-383.el8.x86_64 2/2 2022-05-05T14:11:30.388 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-4.18.0-383.el8.x86_64 2/2 2022-05-05T14:11:30.389 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-4.18.0-383.el8.x86_64 1/2 2022-05-05T14:11:30.864 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-4.18.0-383.el8.x86_64 2/2 2022-05-05T14:11:30.865 INFO:teuthology.orchestra.run.smithi022.stdout: 2022-05-05T14:11:30.865 INFO:teuthology.orchestra.run.smithi022.stdout:Reinstalled: 2022-05-05T14:11:30.865 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-4.18.0-383.el8.x86_64 2022-05-05T14:11:30.866 INFO:teuthology.orchestra.run.smithi022.stdout: 2022-05-05T14:11:30.866 INFO:teuthology.orchestra.run.smithi022.stdout:Complete! 2022-05-05T14:11:30.939 DEBUG:teuthology.orchestra.run.smithi022:> rpm -q kernel --last | head -n 1 2022-05-05T14:11:31.066 INFO:teuthology.orchestra.run.smithi022.stdout:kernel-4.18.0-383.el8.x86_64 Thu 05 May 2022 02:11:29 PM UTC 2022-05-05T14:11:31.067 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-383.el8.x86_64 2022-05-05T14:11:31.067 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2022-05-05T14:11:31.068 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2022-05-05T14:11:31.068 DEBUG:teuthology.orchestra.run.smithi022:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2022-05-05T14:11:31.098 INFO:teuthology.orchestra.run.smithi022.stdout:ttyS1 2022-05-05T14:11:31.114 DEBUG:teuthology.parallel:result is None 2022-05-05T14:11:31.114 INFO:teuthology.run_tasks:Running task internal.base... 2022-05-05T14:11:31.123 INFO:teuthology.task.internal:Creating test directory... 2022-05-05T14:11:31.124 DEBUG:teuthology.orchestra.run.smithi022:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2022-05-05T14:11:31.157 DEBUG:teuthology.orchestra.run.smithi167:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2022-05-05T14:11:31.177 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2022-05-05T14:11:31.188 INFO:teuthology.run_tasks:Running task internal.archive... 2022-05-05T14:11:31.201 INFO:teuthology.task.internal:Creating archive directory... 2022-05-05T14:11:31.202 DEBUG:teuthology.orchestra.run.smithi022:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2022-05-05T14:11:31.214 DEBUG:teuthology.orchestra.run.smithi167:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2022-05-05T14:11:31.251 INFO:teuthology.run_tasks:Running task internal.coredump... 2022-05-05T14:11:31.261 INFO:teuthology.task.internal:Enabling coredump saving... 2022-05-05T14:11:31.262 DEBUG:teuthology.orchestra.run.smithi022:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2022-05-05T14:11:31.276 DEBUG:teuthology.orchestra.run.smithi167:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2022-05-05T14:11:31.306 INFO:teuthology.orchestra.run.smithi022.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-05-05T14:11:31.317 INFO:teuthology.orchestra.run.smithi022.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-05-05T14:11:31.334 INFO:teuthology.orchestra.run.smithi167.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-05-05T14:11:31.346 INFO:teuthology.orchestra.run.smithi167.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-05-05T14:11:31.348 INFO:teuthology.run_tasks:Running task internal.sudo... 2022-05-05T14:11:31.356 INFO:teuthology.task.internal:Configuring sudo... 2022-05-05T14:11:31.357 DEBUG:teuthology.orchestra.run.smithi022:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2022-05-05T14:11:31.361 DEBUG:teuthology.orchestra.run.smithi167:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2022-05-05T14:11:31.390 INFO:teuthology.run_tasks:Running task internal.syslog... 2022-05-05T14:11:31.402 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2022-05-05T14:11:31.402 DEBUG:teuthology.orchestra.run.smithi022:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2022-05-05T14:11:31.428 DEBUG:teuthology.orchestra.run.smithi167:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2022-05-05T14:11:31.448 DEBUG:teuthology.orchestra.run.smithi022:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2022-05-05T14:11:31.524 DEBUG:teuthology.orchestra.run.smithi022:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2022-05-05T14:11:31.586 DEBUG:teuthology.orchestra.run.smithi022:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2022-05-05T14:11:31.668 DEBUG:teuthology.orchestra.run.smithi022:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2022-05-05T14:11:31.712 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2022-05-05T14:11:31.713 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2022-05-05T14:11:31.779 DEBUG:teuthology.orchestra.run.smithi167:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2022-05-05T14:11:31.824 DEBUG:teuthology.orchestra.run.smithi167:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2022-05-05T14:11:31.887 DEBUG:teuthology.orchestra.run.smithi167:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2022-05-05T14:11:31.932 DEBUG:teuthology.orchestra.run.smithi167:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2022-05-05T14:11:31.978 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2022-05-05T14:11:31.978 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2022-05-05T14:11:32.046 DEBUG:teuthology.orchestra.run.smithi022:> sudo service rsyslog restart 2022-05-05T14:11:32.049 DEBUG:teuthology.orchestra.run.smithi167:> sudo service rsyslog restart 2022-05-05T14:11:32.127 INFO:teuthology.orchestra.run.smithi022.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2022-05-05T14:11:32.146 INFO:teuthology.orchestra.run.smithi167.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2022-05-05T14:11:32.406 INFO:teuthology.run_tasks:Running task internal.timer... 2022-05-05T14:11:32.414 INFO:teuthology.task.internal:Starting timer... 2022-05-05T14:11:32.415 INFO:teuthology.run_tasks:Running task pcp... 2022-05-05T14:11:32.436 INFO:teuthology.run_tasks:Running task selinux... 2022-05-05T14:11:32.449 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2022-05-05T14:11:32.450 DEBUG:teuthology.orchestra.run.smithi022:> sudo service auditd rotate 2022-05-05T14:11:32.524 INFO:teuthology.orchestra.run.smithi022.stdout:Rotating logs: 2022-05-05T14:11:32.526 DEBUG:teuthology.orchestra.run.smithi167:> sudo service auditd rotate 2022-05-05T14:11:32.584 INFO:teuthology.orchestra.run.smithi167.stdout:Rotating logs: 2022-05-05T14:11:32.586 DEBUG:teuthology.task.selinux:Getting current SELinux state 2022-05-05T14:11:32.586 DEBUG:teuthology.orchestra.run.smithi022:> /usr/sbin/getenforce 2022-05-05T14:11:32.603 INFO:teuthology.orchestra.run.smithi022.stdout:Permissive 2022-05-05T14:11:32.603 DEBUG:teuthology.orchestra.run.smithi167:> /usr/sbin/getenforce 2022-05-05T14:11:32.627 INFO:teuthology.orchestra.run.smithi167.stdout:Permissive 2022-05-05T14:11:32.628 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi022.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi167.front.sepia.ceph.com': 'permissive'} 2022-05-05T14:11:32.628 DEBUG:teuthology.orchestra.run.smithi022:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|comm="sssd"\|comm="sss_cache"\|context=system_u:system_r:NetworkManager_dispatcher_t:s0\|scontext=system_u:system_r:logrotate_t:s0\)' 2022-05-05T14:11:32.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-05-05T14:11:32.672 DEBUG:teuthology.orchestra.run.smithi167:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|comm="sssd"\|comm="sss_cache"\|context=system_u:system_r:NetworkManager_dispatcher_t:s0\|scontext=system_u:system_r:logrotate_t:s0\)' 2022-05-05T14:11:32.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-05-05T14:11:32.700 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2022-05-05T14:11:32.700 DEBUG:teuthology.orchestra.run.smithi022:> sudo /usr/sbin/setenforce permissive 2022-05-05T14:11:32.738 DEBUG:teuthology.orchestra.run.smithi167:> sudo /usr/sbin/setenforce permissive 2022-05-05T14:11:32.766 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2022-05-05T14:11:32.776 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master was just updated or references a specific commit; assuming it is current 2022-05-05T14:11:32.777 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master to origin/master 2022-05-05T14:11:32.794 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2022-05-05T14:11:32.796 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi022.front.sepia.ceph.com,smithi167.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/cephlab.yml 2022-05-05T14:14:26.440 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2022-05-05T14:14:26.442 DEBUG:teuthology.task.console_log:Killing console logger for smithi022 2022-05-05T14:14:26.443 DEBUG:teuthology.task.console_log:Killing console logger for smithi167 2022-05-05T14:14:26.444 DEBUG:teuthology.exit:Finished running handlers