Found 24 jobs Found 5 distinct failure reasons Failure: Command failed (workunit test suites/fsstress.sh) on smithi173 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=b7a4754d941f016678ca17d548fafb10e7444735 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh' 2 jobs: ['6138441', '6138457'] suites intersection: ['begin', 'ceph-thrash/default', 'clusters/3-mds-2-standby', 'conf/{client', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'msgr-failures/osd-mds-delay', 'osd}', 'overrides/{distro/rhel/{k-distro', 'overrides/{fuse-default-perm-no', 'rhel_latest}', 'session_timeout', 'tasks/cfuse_workunit_suites_fsstress}', 'thrash/{frag_enable', 'thrash_debug}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] suites union: ['begin', 'ceph-thrash/default', 'clusters/3-mds-2-standby', 'conf/{client', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'msgr-failures/osd-mds-delay', 'multimds/thrash/{0-supported-random-distro$/{centos_latest}', 'multimds/thrash/{0-supported-random-distro$/{ubuntu_latest}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/filestore-xfs', 'osd}', 'overrides/{distro/rhel/{k-distro', 'overrides/{fuse-default-perm-no', 'rhel_latest}', 'session_timeout', 'tasks/cfuse_workunit_suites_fsstress}', 'thrash/{frag_enable', 'thrash_debug}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] Failure: "2021-05-27 23:07:40.559002 mon.a (mon.0) 212 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log 1 jobs: ['6138435'] suites: ['begin', 'clusters/9-mds', 'conf/{client', 'fuse-default-perm-no}', 'inline/yes', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'multimds/basic/{0-supported-random-distro$/{ubuntu_latest}', 'objectstore-ec/bluestore-comp', 'osd}', 'overrides/{basic/{frag_enable', 'overrides/{distro/rhel/{k-distro', 'q_check_counter/check_counter', 'rhel_latest}', 'session_timeout', 'tasks/cfuse_workunit_suites_ffsb}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] Failure: The following counters failed to be set on mds daemons: {'mds.imported', 'mds.exported'} 1 jobs: ['6138443'] suites: ['begin', 'clusters/9-mds', 'conf/{client', 'fuse-default-perm-no}', 'inline/yes', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'multimds/basic/{0-supported-random-distro$/{rhel_7}', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{basic/{frag_enable', 'overrides/{distro/rhel/{k-distro', 'q_check_counter/check_counter', 'rhel_latest}', 'session_timeout', 'tasks/cephfs_test_exports}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] Failure: reached maximum tries (50) after waiting for 300 seconds 1 jobs: ['6138454'] suites: ['centos_latest', 'clusters/9-mds', 'conf/{client', 'mds', 'mon', 'mon-debug', 'mount/fuse', 'multimds/verify/{begin', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{fuse-default-perm-no', 'session_timeout', 'tasks/cfuse_workunit_suites_dbench', 'validater/valgrind}', 'verify/{frag_enable', 'whitelist_health', 'whitelist_wrongly_marked_down}}'] Failure: 1 jobs: ['6138438'] suites: ['centos_latest', 'clusters/9-mds', 'conf/{client', 'mds', 'mon', 'mon-debug', 'mount/fuse', 'multimds/verify/{begin', 'objectstore-ec/filestore-xfs', 'osd}', 'overrides/{fuse-default-perm-no', 'session_timeout', 'tasks/cfuse_workunit_suites_fsstress', 'validater/valgrind}', 'verify/{frag_enable', 'whitelist_health', 'whitelist_wrongly_marked_down}}']