Found 61 jobs Found 7 distinct failure reasons Failure: Command failed on smithi141 with status 1: 'sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 74c6e664-9724-11ee-95a3-87774f69a715 --force' 27 jobs: ['7485720', '7485751', '7485773', '7485740', '7485738', '7485762', '7485755', '7485750', '7485769', '7485730', '7485752', '7485741', '7485721', '7485726', '7485766', '7485775', '7485780', '7485756', '7485774', '7485732', '7485727', '7485745', '7485778', '7485760', '7485763', '7485744', '7485725'] suites intersection: [] suites union: ['0-nvme-loop', '1-start', '2-ops/repave-all}', '2-ops/rm-zap-wait}', '2-services/basic', '2-services/iscsi', '2-services/nfs', '2-services/nfs-haproxy-proto', '2-services/nfs-ingress', '2-services/nfs-ingress2', '2-services/nfs2', '2-services/rgw', '2-services/rgw-ingress', '2-thrash', '3-final}', '3-tasks/small-objects', '3-tasks/snaps-few-objects', 'agent/off', 'agent/on', 'fixed-2', 'mode/packaged', 'mode/root', 'mon_election/classic', 'mon_election/connectivity', 'msgr/async', 'msgr/async-v2only', 'orch:cephadm/osds/{0-distro/centos_9.stream_runc', 'orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc', 'orch:cephadm/smoke-singlehost/{0-random-distro$/{centos_9.stream_runc}', 'orch:cephadm/smoke-small/{0-distro/centos_9.stream_runc', 'orch:cephadm/smoke/{0-distro/centos_9.stream_runc', 'orch:cephadm/thrash/{0-distro/centos_9.stream_runc', 'orch:cephadm/with-work/{0-distro/centos_9.stream', 'orch:cephadm/with-work/{0-distro/centos_9.stream_runc', 'orch:cephadm/workunits/{0-distro/centos_9.stream_runc', 'root}', 'start', 'start}', 'task/test_ca_signed_key}', 'task/test_iscsi_container/{centos_9.stream', 'task/test_orch_cli_mon}', 'task/test_orch_cli}', 'task/test_set_mon_crush_locations}', 'tasks/rados_python}', 'tasks/rotate-keys}', 'test_iscsi_container}}'] Failure: Command failed (workunit test cephadm/test_repos.sh) on smithi157 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=479115eb9bf3e8193466e644bc7bebe1146f4825 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_repos.sh' 2 jobs: ['7485777', '7485743'] suites intersection: ['agent/on', 'task/test_cephadm_repos}'] suites union: ['agent/on', 'mon_election/classic', 'mon_election/connectivity', 'orch:cephadm/workunits/{0-distro/centos_9.stream', 'orch:cephadm/workunits/{0-distro/centos_9.stream_runc', 'task/test_cephadm_repos}'] Failure: Command failed on smithi002 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:479115eb9bf3e8193466e644bc7bebe1146f4825 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4b8e7086-9725-11ee-95a3-87774f69a715 -- ceph rgw realm bootstrap -i -' 1 jobs: ['7485729'] suites: ['agent/off', 'mon_election/classic', 'orch:cephadm/workunits/{0-distro/centos_9.stream', 'task/test_rgw_multisite}'] Failure: Command failed on smithi055 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fd6f976-9725-11ee-95a3-87774f69a715 -e sha1=479115eb9bf3e8193466e644bc7bebe1146f4825 -- bash -c \'ceph versions | jq -e \'"\'"\'.overall | length == 1\'"\'"\'\'' 6 jobs: ['7485767', '7485736', '7485771', '7485728', '7485747', '7485722'] suites intersection: [] suites union: ['1-bootstrap/17.2.0', '1-ranks/1', '1-ranks/2', '1-start', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client', '2-nfs', '2-repo_digest/repo_digest', '3-inline/no', '3-inline/yes', '3-upgrade-mgr-staggered', '3-upgrade-with-workload', '3-upgrade/simple', '4-config-upgrade/{fail_fs}', '4-final}', '4-verify}', '4-wait', '5-upgrade-ls', '5-upgrade-with-workload', '6-verify}}', 'agent/off', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fail_fs/yes', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon_election/classic}', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream', 'orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy'] Failure: Command failed (workunit test cephadm/test_adoption.sh) on smithi077 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=479115eb9bf3e8193466e644bc7bebe1146f4825 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_adoption.sh' 1 jobs: ['7485734'] suites: ['agent/off', 'mon_election/classic', 'orch:cephadm/workunits/{0-distro/centos_9.stream_runc', 'task/test_adoption}'] Failure: Command failed (workunit test suites/fsstress.sh) on smithi107 with status 126: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=479115eb9bf3e8193466e644bc7bebe1146f4825 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh' 2 jobs: ['7485779', '7485758'] suites intersection: ['1-ranks/2', '1-volume/{0-create', '2-client', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy'] suites union: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy'] Failure: Failed to reconnect to smithi149 1 jobs: ['7485742'] suites: ['2-repo_digest/defaut', '3-upgrade/staggered', '4-wait', '5-upgrade-ls', 'agent/on', 'mon_election/classic}', 'orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream']