Found 80 jobs Found 4 distinct failure reasons Failure: "2021-01-05T20:51:41.464105+0000 mds.b (mds.0) 79 : cluster [WRN] Scrub error on inode 0x1000000ccbb (/client.0/tmp/payload.1/multiple_rsync_payload.127805/python2.7/dist-packages/matplotlib) see mds.b log and `damage ls` output for details" in cluster log 11 jobs: ['5757605', '5757599', '5757608', '5757640', '5757594', '5757596', '5757601', '5757610', '5757646', '5757642', '5757606'] suites intersection: ['clusters/1a2s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{begin', 'mds', 'mon', 'osd-asserts', 'osd}', 'overrides/{frag_enable', 'scrub/yes', 'session_timeout', 'tasks/workunit/fs/misc}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] suites union: ['clusters/1a2s-mds-1c-client-3node', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs:workload/{begin', 'k-testing}', 'mds', 'mon', 'mount/fuse', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore-ec/filestore-xfs', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{frag_enable', 'rhel_8}', 'scrub/yes', 'session_timeout', 'tasks/workunit/fs/misc}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] Failure: "2021-01-05T20:59:49.509011+0000 mds.b (mds.0) 64 : cluster [WRN] Scrub error on inode 0x1000000f09c (/client.0/tmp/payload.1/multiple_rsync_payload.139872/modules/4.18.0-240.1.1.el8_3.x86_64/kernel/net/netfilter) see mds.b log and `damage ls` output for details" in cluster log 3 jobs: ['5757633', '5757648', '5757604'] suites intersection: ['clusters/1a2s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{begin', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'osd-asserts', 'osd}', 'overrides/{frag_enable', 'scrub/yes', 'session_timeout', 'tasks/workunit/fs/misc}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] suites union: ['clusters/1a2s-mds-1c-client-3node', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'fs:workload/{begin', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-ec-root', 'objectstore-ec/filestore-xfs', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{frag_enable', 'rhel_8}', 'scrub/yes', 'session_timeout', 'tasks/workunit/fs/misc}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] Failure: "2021-01-05T21:19:19.420454+0000 mds.b (mds.0) 94 : cluster [WRN] Scrub error on inode 0x10000015811 (/client.0/tmp/payload.2/multiple_rsync_payload.146797/modules/4.18.0-240.1.1.el8_3.x86_64/kernel/net/netfilter) see mds.b log and `damage ls` output for details" in cluster log 2 jobs: ['5757631', '5757602'] suites intersection: ['clusters/1a2s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{begin', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'osd-asserts', 'osd}', 'overrides/{frag_enable', 'scrub/yes', 'session_timeout', 'tasks/workunit/fs/misc}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] suites union: ['clusters/1a2s-mds-1c-client-3node', 'conf/{client', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs:workload/{begin', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{frag_enable', 'rhel_8}', 'scrub/yes', 'session_timeout', 'tasks/workunit/fs/misc}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] Failure: "2021-01-05T21:33:35.049969+0000 mds.c (mds.0) 44 : cluster [WRN] Scrub error on inode 0x10000000265 (/client.0/tmp/testdir/dir1/dir2/dir3) see mds.c log and `damage ls` output for details" in cluster log 2 jobs: ['5757639', '5757632'] suites intersection: ['clusters/1a2s-mds-1c-client-3node', 'conf/{client', 'fs:workload/{begin', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{frag_enable', 'scrub/yes', 'session_timeout', 'tasks/workunit/fs/misc}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] suites union: ['clusters/1a2s-mds-1c-client-3node', 'conf/{client', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs:workload/{begin', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{frag_enable', 'rhel_8}', 'scrub/yes', 'session_timeout', 'tasks/workunit/fs/misc}', 'whitelist_health', 'whitelist_wrongly_marked_down}']