Found 360 jobs Found 8 distinct failure reasons Failure: timeout expired in wait_until_healthy 67 jobs suites intersection: [] suites union: ['1-start', '2-services/basic', '2-services/client-keyring', '2-services/iscsi', '2-services/mirror', '2-services/nfs', '2-services/nfs-ingress', '2-services/nfs-ingress2', '2-services/nfs2', '2-services/rgw', '2-services/rgw-ingress', '2-thrash', '3-final}', '3-tasks/rados_api_tests', '3-tasks/radosbench', '3-tasks/small-objects', '3-tasks/snaps-few-objects', 'fixed-2', 'mode/packaged', 'mode/root', 'mon_election/classic', 'mon_election/connectivity', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'rados/cephadm/smoke-roleless/{0-distro/centos_8.3_kubic_stable', 'rados/cephadm/smoke-roleless/{0-distro/rhel_8.3_kubic_stable', 'rados/cephadm/smoke-roleless/{0-distro/ubuntu_20.04', 'rados/cephadm/smoke-singlehost/{0-distro$/{centos_8.3_kubic_stable}', 'rados/cephadm/smoke-singlehost/{0-distro$/{rhel_8.3_kubic_stable}', 'rados/cephadm/smoke-singlehost/{0-distro$/{ubuntu_20.04}', 'rados/cephadm/smoke/{distro/centos_8.3_kubic_stable', 'rados/cephadm/smoke/{distro/rhel_8.3_kubic_stable', 'rados/cephadm/smoke/{distro/ubuntu_20.04', 'rados/cephadm/thrash/{0-distro/centos_8.2_kubic_stable', 'rados/cephadm/with-work/{0-distro/centos_8.3_kubic_stable', 'rados/cephadm/with-work/{0-distro/rhel_8.3_kubic_stable', 'rados/cephadm/with-work/{0-distro/ubuntu_20.04', 'root}', 'start', 'start}', 'tasks/rados_api_tests}', 'tasks/rados_python}'] Dead: b'2021-07-17T15:50:44.950 DEBUG:teuthology.exit:Finished running handlers' 17 jobs: ['6277113', '6277021', '6277294', '6276988', '6277153', '6277326', '6277169', '6277253', '6277075', '6277204', '6277098', '6277279', '6277148', '6276975', '6277040', '6277156', '6277219'] suites intersection: [] suites union: ['0-start', '1-bootstrap/16.2.4', '1-bootstrap/16.2.5', '1-bootstrap/octopus', '1-start', '1-tasks', '2-nfs', '2-repo_digest/defaut', '2-repo_digest/repo_digest', '3-start-upgrade', '3-upgrade-with-workload', '4-final}', '4-wait', 'cluster/{12-osds', 'mon_election/classic', 'mon_election/classic}', 'mon_election/connectivity', 'mon_election/connectivity}', 'msgr-failures/few', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rados', 'rados/cephadm/mgr-nfs-upgrade/{0-centos_8.2_kubic_stable', 'rados/cephadm/upgrade/{1-start-distro/1-start-centos_8.3-octopus', 'rados/cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04', 'rados/cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04-15.2.9', 'rados/thrash-erasure-code-big/{ceph', 'rados/upgrade/parallel/{0-distro$/{rhel_8.3_kubic_stable}', 'rados/upgrade/parallel/{0-distro$/{ubuntu_20.04}', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'recovery-overrides/{more-async-partial-recovery}', 'supported-random-distro$/{centos_8.stream}', 'test_rbd_api', 'test_rbd_python}}', 'thrashers/mapgap', 'thrashosds-health', 'upgrade-sequence', 'workload/{ec-rados-default', 'workloads/ec-rados-plugin=jerasure-k=4-m=2}'] Failure: Command failed (workunit test scrub/osd-recovery-scrub.sh) on smithi017 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=033fd16802e8e9dfc1c76c597b0c0af6c518c7e2 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/scrub/osd-recovery-scrub.sh' 1 jobs: ['6277105'] suites: ['rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'workloads/scrub}'] Failure: Command failed (workunit test cephadm/test_cephadm.sh) on smithi151 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=033fd16802e8e9dfc1c76c597b0c0af6c518c7e2 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh' 4 jobs: ['6277268', '6277181', '6277086', '6277000'] suites intersection: ['rados/cephadm/workunits/{0-distro/centos_8.2_kubic_stable', 'task/test_cephadm}'] suites union: ['mon_election/classic', 'mon_election/connectivity', 'rados/cephadm/workunits/{0-distro/centos_8.2_kubic_stable', 'task/test_cephadm}'] Failure: Command failed (workunit test cephadm/test_dashboard_e2e.sh) on smithi089 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=033fd16802e8e9dfc1c76c597b0c0af6c518c7e2 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_dashboard_e2e.sh' 2 jobs: ['6277055', '6277235'] suites intersection: ['debug/mgr', 'rados/dashboard/{centos_8.2_kubic_stable', 'tasks/e2e}'] suites union: ['debug/mgr', 'mon_election/classic', 'mon_election/connectivity', 'rados/dashboard/{centos_8.2_kubic_stable', 'random-objectstore$/{bluestore-comp-snappy}', 'random-objectstore$/{bluestore-stupid}', 'tasks/e2e}'] Failure: reached maximum tries (800) after waiting for 4800 seconds 1 jobs: ['6277123'] suites: ['clusters/{fixed-2', 'fast/fast', 'mon_election/connectivity', 'msgr-failures/few', 'objectstore/bluestore-comp-snappy', 'openstack}', 'rados', 'rados/thrash-erasure-code/{ceph', 'recovery-overrides/{more-active-recovery}', 'supported-random-distro$/{centos_8}', 'thrashers/default', 'thrashosds-health', 'workloads/ec-radosbench}'] Failure: Test failure: test_cluster_info (tasks.cephfs.test_nfs.TestNFS) 4 jobs: ['6277048', '6277316', '6277227', '6277134'] suites intersection: ['rados/cephadm/workunits/{0-distro/centos_8.2_kubic_stable', 'task/test_orch_cli}'] suites union: ['mon_election/classic', 'mon_election/connectivity', 'rados/cephadm/workunits/{0-distro/centos_8.2_kubic_stable', 'task/test_orch_cli}'] Timeout 3h running clone.client.0/qa/workunits/rados/test.sh 1 jobs: ['6277176'] suites: ['clusters/3-mons', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-low-osd-mem-target', 'rados', 'rados/monthrash/{ceph', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/one', 'workloads/rados_api_tests}']