Found 430 jobs Found 9 distinct failure reasons Failure: Cannot connect to remote host smithi141 11 jobs: ['7231989', '7231992', '7231882', '7231960', '7231809', '7232066', '7232203', '7232067', '7232095', '7232201', '7232149'] suites intersection: [] suites union: ['0-kubeadm', '0-nvme-loop', '1-install/mimic', '1-install/mimic-v1only', '1-install/nautilus', '1-install/octopus', '1-rook', '1-start', '2-ops/rm-zap-add}', '2-ops/rm-zap-flag}', '2-partial-upgrade/firsthalf', '2-services/nfs-ingress-rgw-user', '2-services/nfs2', '2-workload/radosbench', '3-final', '3-final}', '3-thrash/default', '4-workload/{rbd-cls', '5-workload/{radosbench', '6-finish-upgrade', '7-pacific', '8-workload/{rbd-python', 'backoff/peering', 'backoff/peering_and_degraded', 'bluestore-bitmap', 'ceph', 'cluster/1-node', 'clusters/{openstack', 'd-balancer/crush-compat', 'd-balancer/on', 'distro$/{ubuntu_18.04}', 'fixed-2', 'k8s/1.21', 'mode/packaged', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/osd-delay', 'msgr/async-v1only', 'net/calico', 'rados', 'rados/cephadm/osds/{0-distro/ubuntu_18.04', 'rados/cephadm/smoke-roleless/{0-distro/ubuntu_18.04', 'rados/cephadm/thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size', 'rados/cephadm/thrash-old-clients/{0-size-min-size-overrides/3-size-2-min-size', 'rados/cephadm/with-work/{0-distro/ubuntu_18.04', 'rados/rook/smoke/{0-distro/ubuntu_18.04', 'rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack', 'rbd-import-export', 'rbd_api}', 'readwrite', 'rook/1.6.2}', 'snaps-few-objects}', 'snaps-many-objects}', 'start', 'start}', 'tasks/rados_api_tests}', 'thrashers/default', 'thrashers/morepggrow', 'thrashers/none', 'thrashosds-health', 'three-plus-one}', 'ubuntu_18.04}', 'workloads/radosbench}', 'workloads/snaps-few-objects}', 'workloads/test_rbd_api}'] Failure: Command failed on smithi195 with status 127: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:5c985a09c75a9cc73e9774aadc248a51b2319e16 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c53d4b2a-d392-11ed-9aff-001a4aab830c -- ceph mon dump -f json' 4 jobs: ['7231900', '7232001', '7232018', '7232213'] suites intersection: [] suites union: ['0-nvme-loop', '1-ranks/1', '1-start', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '2-ops/rmdir-reactivate}', '3-inline/yes', '3-upgrade-with-workload', '4-verify}', '4-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'distro/rhel_8.4_container_tools_rhel8', 'fixed-2', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon_election/connectivity', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'rados/cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'rados/cephadm/osds/{0-distro/rhel_8.4_container_tools_rhel8', 'rados/cephadm/smoke/{0-nvme-loop', 'roles', 'start}', 'tasks/{0-from/v16.2.4'] Failure: Command failed on smithi006 with status 1: 'sudo yum install -y kernel' 14 jobs: ['7231800', '7231860', '7231969', '7232226', '7232094', '7232010', '7232028', '7232173', '7231902', '7231779', '7232111', '7232033', '7232024', '7232085'] suites intersection: [] suites union: ['0-kubeadm', '0-nvme-loop', '1-ranks/1', '1-ranks/2', '1-rook', '1-start', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client', '2-services/nfs2', '2-workload/none', '2-workload/radosbench', '3-final', '3-final}', '3-inline/no', '3-upgrade-with-workload', '4-verify}', '4-verify}}', 'centos_8.stream_container_tools', 'ceph', 'cluster/1-node', 'cluster/3-node', 'clusters/{fixed-2', 'clusters/{fixed-4', 'conf/{client', 'd-thrash/default/{default', 'd-thrash/none', 'ignorelist_wrongly_marked_down', 'k8s/1.21', 'mds', 'mon', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-dispatch-delay', 'msgr/async-v2only', 'net/calico', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-hybrid', 'openstack}', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'rados', 'rados/cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'rados/cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools', 'rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools', 'rados/rook/smoke/{0-distro/ubuntu_18.04', 'rados/rook/smoke/{0-distro/ubuntu_20.04', 'rados/thrash-erasure-code-isa/{arch/x86_64', 'rados/thrash-erasure-code-shec/{ceph', 'rados/verify/{centos_latest', 'recovery-overrides/{default}', 'recovery-overrides/{more-active-recovery}', 'roles', 'rook/1.6.2}', 'rook/master}', 'supported-random-distro$/{centos_8}', 'task/test_nfs}', 'task/test_orch_cli_mon}', 'tasks/rados_api_tests', 'tasks/{0-from/v16.2.4', 'thrashers/default', 'thrashosds-health', 'thrashosds-health}', 'validater/lockdep}', 'validater/valgrind}', 'workloads/ec-rados-plugin=isa-k=2-m=1}', 'workloads/ec-rados-plugin=shec-k=4-m=3-c=2}'] Failure: No module named 'tasks' 6 jobs: ['7232078', '7232077', '7232074', '7232076', '7232031', '7232075'] suites intersection: [] suites union: ['1-bootstrap/16.2.4', '1-pg-log-overrides/short_pg_log', '1-ranks/1', '1-start', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client', '2-nfs', '2-recovery-overrides/{more-async-partial-recovery}', '2-repo_digest/repo_digest', '3-inline/no', '3-scrub-overrides/{default}', '3-upgrade-with-workload', '3-upgrade/staggered', '4-final}', '4-verify}', '4-verify}}', '4-wait', '5-upgrade-ls', 'backoff/peering_and_degraded', 'centos_8.stream_container_tools', 'ceph', 'clusters/{fixed-2', 'conf/{client', 'crc-failures/default', 'd-balancer/on', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon_election/connectivity', 'mon_election/connectivity}', 'msgr-failures/fastclose', 'msgr-failures/many', 'msgr/async', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-snappy', 'openstack', 'openstack}', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'rados', 'rados/basic/{ceph', 'rados/cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'rados/cephadm/mgr-nfs-upgrade/{0-distro/centos_8.stream_container_tools', 'rados/cephadm/upgrade/{1-start-distro/1-start-centos_8.stream_container-tools', 'rados/perf/{ceph', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'roles', 'scheduler/dmclock_1Shard_16Threads', 'settings/optimized', 'supported-random-distro$/{centos_8}', 'tasks/readwrite}', 'tasks/{0-from/v16.2.4', 'thrashers/pggrow', 'thrashosds-health', 'ubuntu_latest', 'workloads/radosbench-high-concurrency}', 'workloads/sample_radosbench}'] Dead: b'2023-04-05T18:49:29.229 DEBUG:teuthology.exit:Finished running handlers' 5 jobs: ['7231756', '7231790', '7231816', '7232150', '7232220'] suites intersection: [] suites union: ['1-bootstrap/16.2.4', '1-bootstrap/16.2.5', '1-bootstrap/octopus', '1-ranks/1', '1-start', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client', '2-nfs', '3-inline/yes', '3-upgrade-with-workload', '4-final}', '4-verify}', '4-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'rados/cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'rados/cephadm/mgr-nfs-upgrade/{0-distro/centos_8.stream_container_tools', 'rados/objectstore/{backends/objectstore', 'roles', 'supported-random-distro$/{ubuntu_latest}}', 'tasks/{0-from/v16.2.4'] Failure: Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=5c985a09c75a9cc73e9774aadc248a51b2319e16 2 jobs: ['7232180', '7231966'] suites intersection: ['rados/cephadm/dashboard/{0-distro/ignorelist_health', 'task/test_e2e}'] suites union: ['rados/cephadm/dashboard/{0-distro/ignorelist_health', 'task/test_e2e}'] Failure: SELinux denials found on ubuntu@smithi157.front.sepia.ceph.com: ['type=AVC msg=audit(1680696989.203:19183): avc: denied { ioctl } for pid=156875 comm="iptables" path="/var/lib/containers/storage/overlay/8f2b428fa42adcf0bcddfb47c2f33c49aa40eb8c0c4f51916180e2ed7f48c3b3/merged" dev="overlay" ino=3412088 scontext=system_u:system_r:iptables_t:s0 tcontext=system_u:object_r:container_file_t:s0:c1022,c1023 tclass=dir permissive=1'] 3 jobs: ['7231994', '7232208', '7232104'] suites intersection: ['rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools', 'task/test_cephadm}'] suites union: ['mon_election/classic', 'mon_election/connectivity', 'rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools', 'task/test_cephadm}'] Failure: {'smithi142.front.sepia.ceph.com': {'changed': True, 'msg': 'All items completed', 'results': [{'_ansible_item_label': 'ceph/python3-asyncssh', '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': True, 'cmd': ['dnf', '-y', 'copr', 'enable', 'ceph/python3-asyncssh'], 'delta': '0:00:00.854185', 'end': '2023-04-05 14:03:05.904790', 'failed': True, 'invocation': {'module_args': {'_raw_params': 'dnf -y copr enable ceph/python3-asyncssh', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'item': 'ceph/python3-asyncssh', 'msg': 'non-zero return code', 'rc': 1, 'start': '2023-04-05 14:03:05.050605', 'stderr': "Enabling a Copr repository. Please note that this repository is not part\nof the main distribution, and quality may vary.\n\nThe Fedora Project does not exercise any power over the contents of\nthis repository beyond the rules outlined in the Copr FAQ at\n,\nand packages are not held to any quality or security level.\n\nPlease do not file bug reports about these packages in Fedora\nBugzilla. In case of problems, contact the owner of this repository.\nCurl error (7): Couldn't connect to server for https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64 [Failed to connect to copr.fedorainfracloud.org port 443: Network is unreachable]", 'stderr_lines': ['Enabling a Copr repository. Please note that this repository is not part', 'of the main distribution, and quality may vary.', '', 'The Fedora Project does not exercise any power over the contents of', 'this repository beyond the rules outlined in the Copr FAQ at', ',', 'and packages are not held to any quality or security level.', '', 'Please do not file bug reports about these packages in Fedora', 'Bugzilla. In case of problems, contact the owner of this repository.', "Curl error (7): Couldn't connect to server for https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64 [Failed to connect to copr.fedorainfracloud.org port 443: Network is unreachable]"], 'stdout': 'Updating Subscription Management repositories.', 'stdout_lines': ['Updating Subscription Management repositories.']}], 'warnings': ["Consider using the dnf module rather than running 'dnf'. If you need to use command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message."]}, 'smithi178.front.sepia.ceph.com': {'changed': True, 'msg': 'All items completed', 'results': [{'_ansible_item_label': 'ceph/python3-asyncssh', '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': True, 'cmd': ['dnf', '-y', 'copr', 'enable', 'ceph/python3-asyncssh'], 'delta': '0:00:00.882348', 'end': '2023-04-05 14:03:07.437822', 'failed': True, 'invocation': {'module_args': {'_raw_params': 'dnf -y copr enable ceph/python3-asyncssh', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'item': 'ceph/python3-asyncssh', 'msg': 'non-zero return code', 'rc': 1, 'start': '2023-04-05 14:03:06.555474', 'stderr': "Enabling a Copr repository. Please note that this repository is not part\nof the main distribution, and quality may vary.\n\nThe Fedora Project does not exercise any power over the contents of\nthis repository beyond the rules outlined in the Copr FAQ at\n,\nand packages are not held to any quality or security level.\n\nPlease do not file bug reports about these packages in Fedora\nBugzilla. In case of problems, contact the owner of this repository.\nCurl error (7): Couldn't connect to server for https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64 [Failed to connect to copr.fedorainfracloud.org port 443: Network is unreachable]", 'stderr_lines': ['Enabling a Copr repository. Please note that this repository is not part', 'of the main distribution, and quality may vary.', '', 'The Fedora Project does not exercise any power over the contents of', 'this repository beyond the rules outlined in the Copr FAQ at', ',', 'and packages are not held to any quality or security level.', '', 'Please do not file bug reports about these packages in Fedora', 'Bugzilla. In case of problems, contact the owner of this repository.', "Curl error (7): Couldn't connect to server for https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64 [Failed to connect to copr.fedorainfracloud.org port 443: Network is unreachable]"], 'stdout': 'Updating Subscription Management repositories.', 'stdout_lines': ['Updating Subscription Management repositories.']}], 'warnings': ["Consider using the dnf module rather than running 'dnf'. If you need to use command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message."]}} 1 jobs: ['7232195'] suites: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-async-recovery}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/normal', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'mon_election/connectivity', 'msgr-failures/osd-delay', 'msgr/async-v1only', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{rhel_8}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/snaps-few-objects-localized}'] Failure: Command failed (workunit test cephadm/test_dashboard_e2e.sh) on smithi099 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=5c985a09c75a9cc73e9774aadc248a51b2319e16 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_dashboard_e2e.sh' 2 jobs: ['7231811', '7232071'] suites intersection: ['rados/cephadm/dashboard/{0-distro/centos_8.stream_container_tools', 'task/test_e2e}'] suites union: ['rados/cephadm/dashboard/{0-distro/centos_8.stream_container_tools', 'task/test_e2e}']