Found 24 jobs Found 3 distinct failure reasons Failure: Command failed on smithi093 with status 128: 'rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone git://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout baa1ea6a9656c3db06c66032fa80b476721947ba' 12 jobs: ['5197874', '5197875', '5197877', '5197860', '5197864', '5197878', '5197862', '5197863', '5197865', '5197859', '5197867', '5197871'] suites intersection: ['conf/{client', 'mds', 'mon', 'osd}', 'whitelist_health'] suites union: ['begin', 'centos_latest', 'ceph-thrash/mds', 'ceph-thrash/mon', 'clusters/3-mds', 'clusters/3-mds-2-standby', 'clusters/9-mds', 'clusters/9-mds-3-standby', 'conf/{client', 'fuse-default-perm-no}', 'inline/no', 'inline/yes', 'k-testing}', 'mds', 'mon', 'mon-debug', 'mount', 'mount/fuse', 'mount/kclient/{k-testing', 'mount/kclient/{mount', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multimds/basic/{0-supported-random-distro$/{centos_8}', 'multimds/basic/{0-supported-random-distro$/{rhel_8}', 'multimds/thrash/{0-supported-random-distro$/{centos_8}', 'multimds/thrash/{0-supported-random-distro$/{rhel_8}', 'multimds/verify/{begin', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore-ec/filestore-xfs', 'osd}', 'overrides/{basic/{frag_enable', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{fuse-default-perm-no', 'q_check_counter/check_counter', 'rhel_8}', 'tasks/cfuse_workunit_norstats}', 'tasks/cfuse_workunit_suites_dbench}', 'tasks/cfuse_workunit_suites_fsstress', 'tasks/cfuse_workunit_suites_fsstress}', 'tasks/cfuse_workunit_suites_fsx}', 'tasks/cfuse_workunit_suites_pjd}', 'thrash/{frag_enable', 'thrash_debug}', 'validater/lockdep}', 'validater/valgrind}', 'verify/{frag_enable', 'whitelist_health', 'whitelist_wrongly_marked_down}', 'whitelist_wrongly_marked_down}}'] Failure: "2020-07-04T06:26:58.282308+0000 mon.b (mon.0) 275 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY)" in cluster log 9 jobs: ['5197870', '5197876', '5197872', '5197858', '5197856', '5197861', '5197879', '5197873', '5197857'] suites intersection: ['conf/{client', 'mds', 'mon', 'osd}', 'whitelist_health'] suites union: ['begin', 'centos_latest', 'ceph-thrash/mon', 'clusters/3-mds', 'clusters/3-mds-2-standby', 'clusters/9-mds', 'conf/{client', 'fuse-default-perm-no}', 'inline/no', 'inline/yes', 'k-testing}', 'mds', 'mon', 'mon-debug', 'mount', 'mount/fuse', 'mount/kclient/{k-testing', 'mount/kclient/{mount', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'msgr-failures/osd-mds-delay', 'multimds/basic/{0-supported-random-distro$/{centos_8}', 'multimds/basic/{0-supported-random-distro$/{rhel_8}', 'multimds/basic/{0-supported-random-distro$/{ubuntu_latest}', 'multimds/thrash/{0-supported-random-distro$/{centos_8}', 'multimds/verify/{begin', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore-ec/filestore-xfs', 'osd}', 'overrides/{basic/{frag_enable', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{fuse-default-perm-no', 'q_check_counter/check_counter', 'rhel_8}', 'tasks/cfuse_workunit_kernel_untar_build}', 'tasks/cfuse_workunit_misc}', 'tasks/cfuse_workunit_suites_blogbench}', 'tasks/cfuse_workunit_suites_dbench', 'tasks/cfuse_workunit_suites_dbench}', 'tasks/cfuse_workunit_suites_ffsb}', 'tasks/cfuse_workunit_suites_fsstress', 'tasks/cfuse_workunit_suites_fsstress}', 'thrash/{frag_enable', 'thrash_debug}', 'validater/lockdep}', 'validater/valgrind}', 'verify/{frag_enable', 'whitelist_health', 'whitelist_wrongly_marked_down}', 'whitelist_wrongly_marked_down}}'] Crash: Command failed on smithi196 with status 42: "cd /home/ubuntu/cephtest && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term env 'OPENSSL_ia32cap=~0x1000000000000000' valgrind --trace-children=no --child-silent-after-fork=yes --num-callers=50 --suppressions=/home/ubuntu/cephtest/valgrind.supp --xml=yes --xml-file=/var/log/ceph/valgrind/mds.d.log --time-stamp=yes --vgdb=yes --exit-on-first-error=yes --error-exitcode=42 --tool=memcheck ceph-mds -f --cluster ceph -i d" ceph version 16.0.0-2962-g5ccfc2d5d3 (5ccfc2d5d3a03d700cf9c4a48b194d7836328981) pacific (dev) 1: (()+0x12dc0) [0xead0dc0] 2: (gsignal()+0x10f) [0xfeb38df] 3: (abort()+0x127) [0xfe9dcf5] 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a9) [0x52d3a53] 5: (()+0x291c1c) [0x52d3c1c] 6: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x1bf) [0x28e08f] 7: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x108) [0x27c888] 8: (DispatchQueue::entry()+0x13da) [0x550e44a] 9: (DispatchQueue::DispatchThread::entry()+0x11) [0x55bf121] 10: (()+0x82de) [0xeac62de] 11: (clone()+0x43) [0xff78133] 1 jobs: ['5197869'] suites: ['centos_latest', 'clusters/9-mds', 'conf/{client', 'mds', 'mon', 'mon-debug', 'mount/fuse', 'multimds/verify/{begin', 'objectstore-ec/filestore-xfs', 'osd}', 'overrides/{fuse-default-perm-no', 'tasks/cfuse_workunit_suites_dbench', 'validater/valgrind}', 'verify/{frag_enable', 'whitelist_health', 'whitelist_wrongly_marked_down}}']