Found 262 jobs Found 8 distinct failure reasons Failure: Command failed on smithi122 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=15.2.16-213-gaa4a6ef6-1bionic ceph-mds=15.2.16-213-gaa4a6ef6-1bionic ceph-common=15.2.16-213-gaa4a6ef6-1bionic ceph-fuse=15.2.16-213-gaa4a6ef6-1bionic ceph-test=15.2.16-213-gaa4a6ef6-1bionic radosgw=15.2.16-213-gaa4a6ef6-1bionic python-ceph=15.2.16-213-gaa4a6ef6-1bionic libcephfs1=15.2.16-213-gaa4a6ef6-1bionic libcephfs-java=15.2.16-213-gaa4a6ef6-1bionic libcephfs-jni=15.2.16-213-gaa4a6ef6-1bionic librados2=15.2.16-213-gaa4a6ef6-1bionic librbd1=15.2.16-213-gaa4a6ef6-1bionic rbd-fuse=15.2.16-213-gaa4a6ef6-1bionic' 1 jobs: ['6832783'] suites: ['1-install/nautilus', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{rbd-cls', '5-workload/{radosbench', '6-finish-upgrade', '7-octopus', '8-workload/{rbd-python', 'bluestore-bitmap', 'rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack', 'rbd-import-export', 'rbd_api}', 'readwrite', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'thrashosds-health', 'ubuntu_18.04}'] Failure: Command failed on smithi089 with status 1: 'sudo yum install -y kernel' 16 jobs: ['6832790', '6832708', '6832781', '6832714', '6832778', '6832752', '6832768', '6832614', '6832775', '6832774', '6832763', '6832759', '6832760', '6832787', '6832754', '6832771'] suites intersection: [] suites union: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-async-recovery}', '3-scrub-overrides/{default}', 'backoff/peering_and_degraded', 'ceph', 'clusters/9-mons', 'clusters/{fixed-2', 'clusters/{fixed-4', 'crc-failures/default', 'd-balancer/upmap', 'd-thrash/default/{default', 'd-thrash/none', 'debug/mgr', 'fast/normal', 'fixed-2', 'mode/root', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/many', 'msgr-failures/osd-delay', 'msgr-failures/osd-dispatch-delay', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'no_pools', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rados', 'rados/basic/{ceph', 'rados/cephadm/with-work/{0-distro/centos_8.stream_container_tools', 'rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools', 'rados/mgr/{clusters/{2-node-mgr}', 'rados/monthrash/{ceph', 'rados/multimon/{clusters/21', 'rados/objectstore/{backends/filestore-idempotent', 'rados/singleton/{all/random-eio', 'rados/singleton/{all/resolve_stuck_peering', 'rados/thrash-erasure-code-shec/{ceph', 'rados/thrash-erasure-code/{ceph', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'rados/verify/{centos_latest', 'recovery-overrides/{more-async-partial-recovery}', 'recovery-overrides/{more-async-recovery}', 'start', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{centos_8}}', 'task/test_cephadm}', 'tasks/failover}', 'tasks/mon_recovery}', 'tasks/rados_api_tests', 'tasks/rados_cls_all', 'tasks/rados_python}', 'tasks/readwrite}', 'thrashers/default', 'thrashers/morepggrow', 'thrashers/none', 'thrashers/one', 'thrashosds-health', 'thrashosds-health}', 'validater/lockdep}', 'validater/valgrind}', 'workloads/cache-agent-big}', 'workloads/ec-rados-plugin=shec-k=4-m=3-c=2}', 'workloads/ec-small-objects-fast-read}', 'workloads/ec-small-objects}', 'workloads/snaps-few-objects}'] Failure: Test failure: test_standby (tasks.mgr.test_dashboard.TestDashboard) 4 jobs: ['6832596', '6832650', '6832715', '6832741'] suites intersection: ['debug/mgr', 'rados/dashboard/{clusters/{2-node-mgr}', 'supported-random-distro$/{rhel_8}', 'tasks/dashboard}'] suites union: ['debug/mgr', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-stupid', 'rados/dashboard/{clusters/{2-node-mgr}', 'supported-random-distro$/{rhel_8}', 'tasks/dashboard}'] Failure: "2022-05-14T09:55:48.349060+0000 osd.3 (osd.3) 1 : cluster [ERR] map e64 had wrong cluster addr ([v2:0.0.0.0:6826/135641,v1:0.0.0.0:6827/135641] != my [v2:172.21.15.146:6826/135641,v1:172.21.15.146:6827/135641])" in cluster log 1 jobs: ['6832682'] suites: ['rados/objectstore/{backends/ceph_objectstore_tool', 'supported-random-distro$/{ubuntu_18.04}}'] Failure: "2022-05-14T08:13:24.232155+0000 osd.3 (osd.3) 51 : cluster [ERR] 3.1es0 deep-scrub : stat mismatch, got 7/7 objects, 3/3 clones, 7/7 dirty, 0/0 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 11272192/11845632 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes." in cluster log 1 jobs: ['6832544'] suites: ['cluster/{12-osds', 'msgr-failures/osd-dispatch-delay', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rados', 'rados/thrash-erasure-code-big/{ceph', 'recovery-overrides/{default}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/careful', 'thrashosds-health', 'workloads/ec-rados-plugin=jerasure-k=4-m=2}'] Timeout 3h running clone.client.0/qa/workunits/mon/test_mon_osdmap_prune.sh 1 jobs: ['6832711'] suites: ['clusters/3-mons', 'msgr-failures/mon-delay', 'msgr/async-v1only', 'objectstore/bluestore-stupid', 'rados', 'rados/monthrash/{ceph', 'supported-random-distro$/{centos_8}', 'thrashers/sync', 'workloads/rados_mon_osdmap_prune}'] Failure: Test failure: test_standby (tasks.mgr.test_prometheus.TestPrometheus) 1 jobs: ['6832694'] suites: ['debug/mgr', 'objectstore/bluestore-low-osd-mem-target', 'rados/mgr/{clusters/{2-node-mgr}', 'supported-random-distro$/{rhel_8}', 'tasks/prometheus}'] Dead: b'2022-05-14T16:19:43.723 DEBUG:teuthology.exit:Finished running handlers' 1 jobs: ['6832699'] suites: ['cluster/{12-osds', 'msgr-failures/few', 'objectstore/bluestore-comp-lz4', 'openstack}', 'rados', 'rados/thrash-erasure-code-big/{ceph', 'recovery-overrides/{more-async-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/pggrow', 'thrashosds-health', 'workloads/ec-rados-plugin=jerasure-k=4-m=2}']