Found 93 jobs Found 7 distinct failure reasons Failure: Test failure: test_cluster_info (tasks.cephfs.test_nfs.TestNFS) 2 jobs: ['7361478', '7361544'] suites intersection: ['agent/on', 'mon_election/connectivity', 'task/test_nfs}'] suites union: ['agent/on', 'mon_election/connectivity', 'orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_rhel8', 'orch:cephadm/workunits/{0-distro/ubuntu_20.04', 'task/test_nfs}'] Failure: Command failed on smithi060 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:52278f137bab78e32337b36683483b1e47e8490a shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7250b382-35a0-11ee-9b37-001a4aab830c -- ceph-volume lvm zap /dev/nvme2n1' 2 jobs: ['7361495', '7361499'] suites intersection: ['0-nvme-loop'] suites union: ['0-nvme-loop', '1-start', '2-services/rgw-ingress', '3-final}', 'agent/on', 'fixed-2', 'mon_election/connectivity', 'orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_3.0', 'orch:cephadm/smoke/{0-distro/ubuntu_20.04', 'start}'] Failure: SSH connection to smithi031 was lost: 'sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c \'echo "Check with $(hostname) ganesha(s) down..."\nfor c in `systemctl | grep ceph- | grep @nfs | awk \'"\'"\'{print $1}\'"\'"\'`; do\n cid=`echo $c | sed \'"\'"\'s/@/-/\'"\'"\'`\n id=`echo $c | cut -d @ -f 2 | sed \'"\'"\'s/.service$//\'"\'"\'`\n fsid=`echo $c | cut -d @ -f 1 | cut -d - -f 2-`\n echo "Removing daemon $id fsid $fsid..."\n sudo $TESTDIR/cephadm rm-daemon --fsid $fsid --name $id\n\n echo "Waking up cephadm..."\n sudo $TESTDIR/cephadm shell -- ceph orch ps --refresh\n\n while ! timeout 1 cat /mnt/foo/testfile ; do true ; done\n echo "Mount is back!"\ndone\n\'' 2 jobs: ['7361549', '7361484'] suites intersection: ['0-nvme-loop', '1-start', '2-services/nfs-ingress2', '3-final}'] suites union: ['0-nvme-loop', '1-start', '2-services/nfs-ingress2', '3-final}', 'orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_3.0', 'orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_rhel8'] Failure: SELinux denials found on ubuntu@smithi053.front.sepia.ceph.com: ['type=AVC msg=audit(1691471066.226:20668): avc: denied { create } for pid=128494 comm="truncate" name="presized.dat" scontext=system_u:system_r:container_t:s0:c90,c187 tcontext=system_u:object_r:ceph_var_lib_t:s0 tclass=file permissive=1', 'type=AVC msg=audit(1691471059.887:20645): avc: denied { write open } for pid=128073 comm="bash" path="/var/cache/bar/from.txt" dev="dm-4" ino=149 scontext=system_u:system_r:container_t:s0:c150,c584 tcontext=system_u:object_r:ceph_var_lib_t:s0 tclass=file permissive=1', 'type=AVC msg=audit(1691471059.887:20645): avc: denied { create } for pid=128073 comm="bash" name="from.txt" scontext=system_u:system_r:container_t:s0:c150,c584 tcontext=system_u:object_r:ceph_var_lib_t:s0 tclass=file permissive=1', 'type=AVC msg=audit(1691471059.887:20645): avc: denied { add_name } for pid=128073 comm="bash" name="from.txt" scontext=system_u:system_r:container_t:s0:c150,c584 tcontext=unconfined_u:object_r:ceph_var_lib_t:s0 tclass=dir permissive=1', 'type=AVC msg=audit(1691471066.226:20668): avc: denied { write } for pid=128494 comm="truncate" name="data" dev="dm-4" ino=148 scontext=system_u:system_r:container_t:s0:c90,c187 tcontext=unconfined_u:object_r:ceph_var_lib_t:s0 tclass=dir permissive=1', 'type=AVC msg=audit(1691471066.226:20668): avc: denied { write open } for pid=128494 comm="truncate" path="/var/cache/bar/presized.dat" dev="dm-4" ino=150 scontext=system_u:system_r:container_t:s0:c90,c187 tcontext=system_u:object_r:ceph_var_lib_t:s0 tclass=file permissive=1', 'type=AVC msg=audit(1691471066.226:20668): avc: denied { add_name } for pid=128494 comm="truncate" name="presized.dat" scontext=system_u:system_r:container_t:s0:c90,c187 tcontext=unconfined_u:object_r:ceph_var_lib_t:s0 tclass=dir permissive=1', 'type=AVC msg=audit(1691471059.887:20645): avc: denied { write } for pid=128073 comm="bash" name="data" dev="dm-4" ino=148 scontext=system_u:system_r:container_t:s0:c150,c584 tcontext=unconfined_u:object_r:ceph_var_lib_t:s0 tclass=dir permissive=1'] 1 jobs: ['7361529'] suites: ['agent/on', 'mon_election/connectivity', 'orch:cephadm/workunits/{0-distro/centos_8.stream_container_tools_crun', 'task/test_extra_daemon_features}'] Failure: SSH connection to smithi050 was lost: 'sudo PATH=/usr/sbin:$PATH lsof ; ps auxf' 2 jobs: ['7361500', '7361528'] suites intersection: ['1-ranks/2', '1-volume/{0-create', '2-client', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/yes', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/pacific'] suites union: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/yes', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/pacific'] Failure: Command failed on smithi152 with status 127: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 743b9118-35a9-11ee-9b37-001a4aab830c -- ceph mon dump -f json' 1 jobs: ['7361555'] suites: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/yes', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/pacific'] Failure: reached maximum tries (301) after waiting for 300 seconds 1 jobs: ['7361524'] suites: ['0-nvme-loop', '1-start', '2-services/jaeger', '3-final}', 'orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools_crun']