Found 112 jobs Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Found 6 distinct failure reasons Failure: Command failed on smithi032 with status 32: "sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mount -t nfs 10.0.31.32:/fake /mnt/foo'" 7 jobs: ['7389893', '7389971', '7389973', '7389879', '7389966', '7389900', '7389906'] suites intersection: [] suites union: ['0-nvme-loop', '1-start', '2-services/nfs-haproxy-proto', '2-services/nfs-ingress', '2-services/nfs-ingress2', '2-services/nfs-keepalive-only', '3-final}', 'agent/on', 'fixed-2', 'mode/root', 'mon_election/connectivity', 'msgr/async', 'orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools', 'orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_3.0', 'orch:cephadm/smoke-roleless/{0-distro/ubuntu_20.04', 'orch:cephadm/with-work/{0-distro/rhel_8.6_container_tools_rhel8', 'orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_rhel8', 'start', 'task/test_orch_cli_mon}', 'tasks/rados_python}'] Failure: reached maximum tries (301) after waiting for 300 seconds 3 jobs: ['7389956', '7389869', '7389947'] suites intersection: ['1-start'] suites union: ['0-nvme-loop', '1-bootstrap/16.2.0', '1-bootstrap/16.2.5', '1-start', '2-nfs', '2-services/jaeger', '3-final}', '3-upgrade-with-workload', '4-final}', 'orch:cephadm/mgr-nfs-upgrade/{0-centos_8.stream_container_tools', 'orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools'] Failure: Command failed on smithi079 with status 123: "sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --" 5 jobs: ['7389924', '7389925', '7389892', '7389964', '7389962'] suites intersection: [] suites union: ['0-nvme-loop', '1-start', '2-services/mirror', '3-final}', 'agent/on', 'fixed-2', 'mode/root', 'mon_election/connectivity', 'msgr/async-v1only', 'msgr/async-v2only', 'orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/smoke/{0-distro/ubuntu_20.04', 'orch:cephadm/with-work/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/with-work/{0-distro/rhel_8.6_container_tools_rhel8', 'orch:cephadm/workunits/{0-distro/ubuntu_20.04', 'start', 'start}', 'task/test_orch_cli_mon}', 'tasks/rados_api_tests}', 'tasks/rotate-keys}'] Failure: Test failure: test_cluster_set_reset_user_config (tasks.cephfs.test_nfs.TestNFS) 3 jobs: ['7389876', '7389963', '7389917'] suites intersection: ['agent/on', 'mon_election/connectivity', 'task/test_nfs}'] suites union: ['agent/on', 'mon_election/connectivity', 'orch:cephadm/workunits/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_3.0', 'orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_rhel8', 'task/test_nfs}'] Failure: Command failed on smithi156 with status 22: "sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6bb5a89a-4c9b-11ee-9ab3-7b867c8bd7da -- bash -c 'ceph orch upgrade ls'" 5 jobs: ['7389875', '7389931', '7389958', '7389902', '7389908'] suites intersection: [] suites union: ['1-bootstrap/16.2.4', '1-start', '2-nfs', '2-repo_digest/defaut', '2-repo_digest/repo_digest', '3-upgrade-with-workload', '3-upgrade/simple', '3-upgrade/staggered', '4-final}', '4-wait', '5-upgrade-ls', 'agent/off', 'agent/on', 'mon_election/classic}', 'mon_election/connectivity}', 'orch:cephadm/mgr-nfs-upgrade/{0-centos_8.stream_container_tools', 'orch:cephadm/upgrade/{1-start-distro/1-start-centos_8.stream_container-tools', 'orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04'] Failure: SELinux denials found on ubuntu@smithi027.front.sepia.ceph.com: ['type=AVC msg=audit(1693999574.691:21265): avc: denied { ioctl } for pid=120263 comm="iptables" path="/var/lib/containers/storage/overlay/fb6484a51c3dfd3bb78793a77565ff4856424b7150bc34279a8130d1470d542d/merged" dev="overlay" ino=2754125 scontext=system_u:system_r:iptables_t:s0 tcontext=system_u:object_r:container_file_t:s0:c1022,c1023 tclass=dir permissive=1'] 1 jobs: ['7389948'] suites: ['agent/off', 'mon_election/classic', 'orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_rhel8', 'task/test_cephadm}']