Found 139 jobs Missing teuthology log /home/teuthworker/archive/yuriw-2023-04-28_19:04:20-orch-reef-distro-default-smithi/7257762/teuthology.log Found 9 distinct failure reasons Failure: Command failed on smithi110 with status 1: 'sudo systemctl enable --now kubelet && sudo kubeadm config images pull' 5 jobs: ['7257655', '7257724', '7257689', '7257759', '7257637'] suites intersection: [] suites union: ['0-kubeadm', '0-nvme-loop', '1-bootstrap/16.2.5', '1-rook', '1-start', '2-nfs', '2-workload/none', '2-workload/radosbench', '3-upgrade-with-workload', '4-final}', 'cluster/1-node', 'cluster/3-node', 'k8s/1.21', 'net/calico', 'net/flannel', 'net/host', 'orch/cephadm/mgr-nfs-upgrade/{0-centos_8.stream_container_tools', 'orch/rook/smoke/{0-distro/ubuntu_20.04', 'rook/1.7.2}', 'rook/master}'] Failure: Command failed on smithi189 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 52363c02-e7e4-11ed-9b00-001a4aab830c -e sha1=565a0d716c29a28798d290d0d9f387d35c9531f3 -- bash -c \'ceph versions | jq -e \'"\'"\'.osd | length == 2\'"\'"\'\'' 3 jobs: ['7257679', '7257721', '7257653'] suites intersection: ['3-upgrade/staggered', '4-wait', '5-upgrade-ls'] suites union: ['2-repo_digest/defaut', '2-repo_digest/repo_digest', '3-upgrade/staggered', '4-wait', '5-upgrade-ls', 'agent/off', 'agent/on', 'mon_election/classic}', 'mon_election/connectivity}', 'orch/cephadm/upgrade/{1-start-distro/1-start-centos_8.stream_container-tools', 'orch/cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04'] Failure: Command failed on smithi105 with status 127: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:565a0d716c29a28798d290d0d9f387d35c9531f3 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5d3790fe-e7e6-11ed-9b00-001a4aab830c -- ceph osd stat -f json' 1 jobs: ['7257686'] suites: ['0-nvme-loop', '1-start', '2-services/nfs', '3-final}', 'orch/cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_3.0'] Failure: Test failure: test_cluster_set_reset_user_config (tasks.cephfs.test_nfs.TestNFS) 1 jobs: ['7257750'] suites: ['agent/on', 'mon_election/connectivity', 'orch/cephadm/workunits/{0-distro/centos_8.stream_container_tools_crun', 'task/test_nfs}'] Dead: None 1 jobs: ['7257762'] Timeout 3h running clone.client.0/qa/workunits/suites/fsstress.sh 1 jobs: ['7257641'] suites: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/no', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'orch/cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/pacific'] Failure: Command failed on smithi177 with status 123: "find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --" 4 jobs: ['7257742', '7257654', '7257677', '7257697'] suites intersection: ['fixed-2'] suites union: ['0-nvme-loop', '1-start', '2-thrash', '3-tasks/radosbench', 'agent/off', 'fixed-2', 'mon_election/connectivity', 'msgr/async-v1only', 'msgr/async-v2only', 'orch/cephadm/smoke/{0-distro/centos_8.stream_container_tools', 'orch/cephadm/smoke/{0-distro/rhel_8.6_container_tools_rhel8', 'orch/cephadm/thrash/{0-distro/rhel_8.6_container_tools_rhel8', 'orch/cephadm/thrash/{0-distro/ubuntu_20.04', 'root}', 'start}'] Failure: Test failure: test_cephfs_mirror (tasks.cephadm_cases.test_cli.TestCephadmCLI) 1 jobs: ['7257702'] suites: ['agent/on', 'mon_election/connectivity', 'orch/cephadm/workunits/{0-distro/centos_8.stream_container_tools', 'task/test_orch_cli}'] Failure: SELinux denials found on ubuntu@smithi151.front.sepia.ceph.com: ['type=AVC msg=audit(1682924372.533:20131): avc: denied { ioctl } for pid=110738 comm="iptables" path="/var/lib/containers/storage/overlay/271e992c3e307e4fb9ffa821e769eaf510be8805b5508d35f4d6aadf6e9d355b/merged" dev="overlay" ino=3280971 scontext=system_u:system_r:iptables_t:s0 tcontext=system_u:object_r:container_file_t:s0:c1022,c1023 tclass=dir permissive=1'] 1 jobs: ['7257725'] suites: ['agent/off', 'mon_election/classic', 'orch/cephadm/workunits/{0-distro/rhel_8.6_container_tools_rhel8', 'task/test_cephadm}']