Found 491 jobs
Found 8 distinct failure reasons
Failure: Command failed on smithi176 with status 22: "sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:fcf4fd271c18bb40990262aa80b79e7e03ed7ad0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0c2291a8-1ab3-11eb-a2ad-001a4aab830c -- ceph orch daemon add mon 'smithi176:[v1:172.21.15.176:6789]=b'"
1 jobs: ['5571790']
suites: ['1-install/mimic', 'backoff/peering', 'ceph', 'clusters/{openstack', 'd-balancer/on', 'distro$/{ubuntu_18.04}', 'mon_election/connectivity', 'msgr-failures/fastclose', 'rados', 'rados/cephadm/thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size', 'thrashers/none', 'thrashosds-health', 'three-plus-one}', 'workloads/rbd_cls}']

Failure: "2020-10-31T15:01:54.378492+0000 osd.11 (osd.11) 84 : cluster [ERR] 5.f deep-scrub : stat mismatch, got 4/4 objects, 0/0 clones, 0/0 dirty, 0/0 omap, 0/0 pinned, 4/4 hit_set_archive, 0/0 whiteouts, 6553/0 bytes, 0/0 manifest objects, 6553/6553 hit_set_archive bytes." in cluster log
1 jobs: ['5572136']
suites: ['1-install/nautilus', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{rbd-cls', '5-workload/{radosbench', '6-finish-upgrade', '7-octopus', '8-workload/{rbd-python', 'bluestore-bitmap', 'mon_election/connectivity', 'rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack', 'rbd-import-export', 'rbd_api}', 'readwrite', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'thrashosds-health', 'ubuntu_latest}']

Timeout 3h running clone.client.0/qa/workunits/mon/caps.sh
2 jobs: ['5571924', '5572211']
suites intersection: ['clusters/3-mons', 'mon_election/classic', 'msgr-failures/mon-delay', 'msgr/async-v1only', 'rados', 'rados/monthrash/{ceph', 'workloads/rados_mon_workunits}']
suites union: ['clusters/3-mons', 'mon_election/classic', 'msgr-failures/mon-delay', 'msgr/async-v1only', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-stupid', 'rados', 'rados/monthrash/{ceph', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}', 'thrashers/force-sync-many', 'thrashers/sync', 'workloads/rados_mon_workunits}']

Failure: Command failed on smithi140 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic'
1 jobs: ['5572151']
suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{more-async-partial-recovery}', 'backoff/peering_and_degraded', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'mon_election/classic', 'msgr-failures/fastclose', 'msgr/async-v1only', 'objectstore/bluestore-bitmap', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/careful', 'thrashosds-health', 'workloads/small-objects}']

Failure: Command failed (workunit test rados/test.sh) on smithi064 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=fcf4fd271c18bb40990262aa80b79e7e03ed7ad0 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test.sh'
1 jobs: ['5571904']
suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'backoff/normal', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'mon_election/classic', 'msgr-failures/few', 'msgr/async-v2only', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/none', 'thrashosds-health', 'workloads/rados_api_tests}']

Dead: b'2020-10-31T01:11:21.546 DEBUG:teuthology.exit:Finished running handlers'
4 jobs: ['5571783', '5572064', '5571975', '5572230']
suites intersection: ['ceph', 'clusters/{fixed-2', 'openstack}', 'rados']
suites union: ['1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{default}', '2-recovery-overrides/{more-partial-recovery}', 'backoff/normal', 'backoff/peering_and_degraded', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'crc-failures/default', 'd-balancer/crush-compat', 'd-balancer/on', 'd-thrash/default/{default', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'rados/verify/{centos_latest', 'supported-random-distro$/{centos_8}', 'tasks/rados_api_tests', 'thrashers/careful', 'thrashers/default', 'thrashosds-health', 'thrashosds-health}', 'validater/valgrind}', 'workloads/redirect_set_object}', 'workloads/snaps-few-objects-balanced}']

Failure: "2020-10-31T12:06:53.878914+0000 mon.b (mon.0) 11 : cluster [WRN] Health check failed: 1/3 mons down, quorum b,a (MON_DOWN)" in cluster log
1 jobs: ['5572103']
suites: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{default}', 'backoff/peering_and_degraded', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/crush-compat', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr/async-v1only', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{centos_8}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/cache-pool-snaps-readproxy}']

Failure: "2020-10-31T11:37:11.704170+0000 osd.0 (osd.0) 325 : cluster [ERR] scrub 57.1b 57:db40e119:test-rados-api-smithi149-21496-74::foo:24 : size 0 != clone_size 10" in cluster log
2 jobs: ['5572048', '5571824']
suites intersection: ['2-recovery-overrides/{more-async-recovery}', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'msgr-failures/osd-delay', 'msgr/async', 'openstack}', 'rados', 'thrashosds-health', 'workloads/rados_api_tests}']
suites union: ['1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-async-recovery}', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'crc-failures/default', 'd-balancer/crush-compat', 'd-balancer/on', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/osd-delay', 'msgr/async', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/mapgap', 'thrashers/pggrow', 'thrashosds-health', 'workloads/rados_api_tests}']