Found 300 jobs
Found 8 distinct failure reasons
Failure: error during scrub thrashing: reached maximum tries (30) after waiting for 900 seconds
12 jobs: ['7002448', '7002357', '7002389', '7002359', '7002373', '7002342', '7002283', '7002387', '7002224', '7002267', '7002434', '7002403']
suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd-asserts', 'osd}', 'overrides/{frag', 'ranks/multi/{export-check', 'session_timeout}', 'standby-replay']
suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/fs/misc}}', '5-workunit/kernel_untar_build}}', '5-workunit/postgres}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/legacy', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{centos_latest', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{no-subvolume}', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}', 'wsync/yes}']

Failure: Command failed (workunit test suites/pjd.sh) on smithi046 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0df3eb9223d9941354f419f47309731f592577a5 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh'
16 jobs: ['7002400', '7002430', '7002370', '7002235', '7002205', '7002460', '7002265', '7002280', '7002385', '7002325', '7002190', '7002220', '7002415', '7002295', '7002310', '7002340']
suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd-asserts', 'osd}', 'overrides/{frag', 'session_timeout}', 'standby-replay']
suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/legacy', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{centos_latest', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/1', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}', 'wsync/yes}']

Dead: b'2022-09-01T01:45:52.386 DEBUG:teuthology.exit:Finished running handlers'
7 jobs: ['7002212', '7002345', '7002285', '7002225', '7002392', '7002253', '7002332']
suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'osd-asserts', 'osd}', 'overrides/{frag', 'ranks/multi/{export-check', 'session_timeout}', 'standby-replay']
suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/kernel_untar_build}}', '5-workunit/suites/blogbench}}', '5-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/legacy', 'n/3', 'n/5', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{centos_latest', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{no-subvolume}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'wsync/no}', 'wsync/yes}']

Failure: Command failed on smithi163 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:0df3eb9223d9941354f419f47309731f592577a5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 897e3a9a-2992-11ed-8431-001a4aab830c -- ceph orch daemon add osd smithi163:vg_nvme/lv_3'
3 jobs: ['7002346', '7002362', '7002182']
suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '4-flush/no', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'ms_mode/secure', 'objectstore-ec/bluestore-bitmap', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/multi/{export-check', 'replication/default}', 'session_timeout}', 'standby-replay', 'wsync/yes}']
suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/no', '5-workunit/suites/dbench}}', '5-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/multi/{export-check', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{no-subvolume}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/yes}']

Failure: Command failed on smithi088 with status 22: "sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule status --fs=cephfs --path=/'"
3 jobs: ['7002437', '7002272', '7002317']
suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/yes', '5-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'n/5', 'osd-asserts', 'osd}', 'overrides/{frag', 'ranks/multi/{export-check', 'session_timeout}', 'standby-replay']
suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/legacy', 'n/5', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{centos_latest', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{no-subvolume}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}', 'wsync/yes}']

Failure: error during scrub thrashing: Command failed on smithi097 with status 13: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status'
2 jobs: ['7002196', '7002464']
suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '4-flush/no', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'ms_mode/crc', 'objectstore-ec/bluestore-comp', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/multi/{export-check', 'session_timeout}', 'standby-replay', 'wsync/yes}']
suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/no', '5-workunit/postgres}}', '5-workunit/suites/dbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'n/3', 'n/5', 'objectstore-ec/bluestore-comp', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/yes}']

Failure: Command failed (workunit test suites/dbench.sh) on smithi088 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0df3eb9223d9941354f419f47309731f592577a5 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/dbench.sh'
4 jobs: ['7002406', '7002226', '7002286', '7002466']
suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '3-snaps/yes', '4-flush/no', '5-workunit/suites/dbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'objectstore-ec/bluestore-bitmap', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/multi/{export-check', 'session_timeout}', 'standby-replay', 'wsync/yes}']
suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '3-snaps/yes', '4-flush/no', '5-workunit/suites/dbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/yes}']

Failure: Command failed (workunit test suites/ffsb.sh) on smithi046 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0df3eb9223d9941354f419f47309731f592577a5 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'
1 jobs: ['7002452']
suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/no', '5-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{0-centos_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'n/5', 'objectstore-ec/bluestore-comp', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/multi/{export-check', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-no-extra-options}', 'wsync/yes}']