Found 356 jobs Found 10 distinct failure reasons Failure: Exiting scrub checking -- not all pgs scrubbed. 4 jobs: ['6004218', '6004252', '6004030', '6003960'] suites intersection: ['ceph', 'clusters/{fixed-2', 'openstack}', 'rados', 'thrashosds-health'] suites union: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-active-recovery}', '2-recovery-overrides/{more-async-partial-recovery}', 'backoff/normal', 'backoff/peering_and_degraded', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/crush-compat', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/osd-delay', 'msgr/async', 'msgr/async-v2only', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/thrash-erasure-code-isa/{arch/x86_64', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'recovery-overrides/{default}', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/morepggrow', 'thrashers/pggrow', 'thrashosds-health', 'workloads/cache-snaps-balanced}', 'workloads/ec-rados-plugin=isa-k=2-m=1}', 'workloads/pool-snaps-few-objects}', 'workloads/radosbench}'] Dead: b'2021-03-28T02:52:19.987 DEBUG:teuthology.exit:Finished running handlers' 4 jobs: ['6004146', '6003994', '6004059', '6004097'] suites intersection: [] suites union: ['1-tasks', 'ceph', 'cluster/{12-osds', 'clusters/{fixed-2', 'd-thrash/default/{default', 'distro1$/{ubuntu_20.04_kubic_testing}', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async-v2only', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rados', 'rados/singleton-nomsgr/{all/lazy_omap_stats_output', 'rados/thrash-erasure-code-big/{ceph', 'rados/upgrade/pacific-x/parallel/{0-start', 'rados/verify/{centos_latest', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'recovery-overrides/{default}', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}}', 'tasks/rados_cls_all', 'test_rbd_api', 'test_rbd_python}}', 'thrashers/mapgap', 'thrashosds-health', 'thrashosds-health}', 'upgrade-sequence', 'validater/valgrind}', 'workload/{ec-rados-default', 'workloads/ec-rados-plugin=jerasure-k=4-m=2}'] Failure: Command failed on gibba009 with status 100: 'sudo apt-get clean' 1 jobs: ['6004043'] suites: ['clusters/{fixed-4', 'mon_election/classic', 'msgr-failures/osd-delay', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/thrash-erasure-code-shec/{ceph', 'recovery-overrides/{more-async-partial-recovery}', 'supported-random-distro$/{rhel_8}', 'thrashers/careful', 'thrashosds-health', 'workloads/ec-rados-plugin=shec-k=4-m=3-c=2}'] Failure: Command failed on gibba021 with status 126: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:3e31def9543c902ee0dbe0a9dd079680aaa5a200 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 29887b1c-8f0d-11eb-ab30-001a4aab830c -- ceph orch host ls --format=json' 2 jobs: ['6004044', '6004029'] suites intersection: [] suites union: ['1-start', '2-services/mirror', '3-final}', 'fixed-2', 'mon_election/classic', 'rados/cephadm/smoke-roleless/{0-distro/ubuntu_20.04_kubic_stable', 'rados/cephadm/smoke/{distro/ubuntu_20.04_kubic_stable', 'start}'] Failure: rgw multisite test failures 1 jobs: ['6004232'] suites: ['frontend', 'overrides', 'rados/upgrade/pacific-x/rgw-multisite/{clusters', 'realm', 'tasks', 'upgrade/primary}'] Failure: SSH connection to gibba009 was lost: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic' 1 jobs: ['6004046'] suites: ['1-start', '2-services/rgw', '3-final}', 'rados/cephadm/smoke-roleless/{0-distro/ubuntu_20.04_kubic_testing'] Dead: b'2021-03-28T00:52:51.313 INFO:journalctl@ceph.mon.b.gibba013.stdout:Mar 28 00:52:50 gibba013 bash[12325]: cluster 2021-03-28T00:52:49.982589+0000 mon.a (mon.0) 50346 : cluster [DBG] mgrmap e4117: y(active, since 11h), standbys: x' 1 jobs: ['6003966'] suites: ['1-tasks', 'distro1$/{ubuntu_20.04}', 'mon_election/connectivity', 'rados/upgrade/pacific-x/parallel/{0-start', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'test_rbd_api', 'test_rbd_python}}', 'upgrade-sequence', 'workload/{ec-rados-default'] Failure: reached maximum tries (50) after waiting for 300 seconds 1 jobs: ['6004133'] suites: ['ceph', 'clusters/{fixed-2', 'd-thrash/default/{default', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-hybrid', 'openstack}', 'rados', 'rados/verify/{centos_latest', 'tasks/mon_recovery', 'thrashosds-health}', 'validater/valgrind}'] Failure: Test failure: test_pwd_expiration_date_update (tasks.mgr.dashboard.test_user.UserTest) 1 jobs: ['6004141'] suites: ['debug/mgr', 'mon_election/classic', 'rados/dashboard/{clusters/{2-node-mgr}', 'random-objectstore$/{bluestore-stupid}', 'supported-random-distro$/{centos_8}', 'tasks/dashboard}'] Failure: Command failed on gibba028 with status 1: "sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump" 1 jobs: ['6004050'] suites: ['clusters/{fixed-2', 'mon_election/classic', 'msgr-failures/many', 'msgr/async-v1only', 'objectstore/bluestore-bitmap', 'openstack}', 'rados', 'rados/basic/{ceph', 'supported-random-distro$/{ubuntu_latest}', 'tasks/libcephsqlite}']