Found 20 jobs Found 3 distinct failure reasons Failure: "2021-02-06T17:05:31.950056+0000 mds.f (mds.0) 14 : cluster [WRN] Scrub error on inode 0x10000000aef (/client.0/tmp/blogbench-1.0/src/blogtest_in/blog-13) see mds.f log and `damage ls` output for details" in cluster log 2 jobs: ['5861439', '5861431'] suites intersection: ['clusters/1a5s-mds-1c-client-3node', 'conf/{client', 'fs/workload/{begin', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{frag_enable', 'rhel_8}', 'scrub/yes', 'session_timeout', 'tasks/{0-check-counter', 'whitelist_health', 'whitelist_wrongly_marked_down}'] suites union: ['clusters/1a5s-mds-1c-client-3node', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'fs/workload/{begin', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{frag_enable', 'ranks/3', 'ranks/5', 'rhel_8}', 'scrub/yes', 'session_timeout', 'tasks/{0-check-counter', 'whitelist_health', 'whitelist_wrongly_marked_down}', 'workunit/fs/misc}}', 'workunit/suites/blogbench}}'] Failure: SELinux denials found on ubuntu@gibba004.front.sepia.ceph.com: ['type=AVC msg=audit(1612634409.975:8472): avc: denied { read write open } for pid=15643 comm="cachefilesd" path="/home/ubuntu/cephtest/archive/coredump/1612634407.15643.core" dev="sda1" ino=524499 scontext=system_u:system_r:cachefilesd_t:s0 tcontext=system_u:object_r:user_home_t:s0 tclass=file permissive=1', 'type=AVC msg=audit(1612634408.842:8468): avc: denied { add_name } for pid=15643 comm="cachefilesd" name="1612634407.15643.core" scontext=system_u:system_r:cachefilesd_t:s0 tcontext=unconfined_u:object_r:user_home_t:s0 tclass=dir permissive=1', 'type=AVC msg=audit(1612634407.838:8463): avc: denied { search } for pid=15643 comm="cachefilesd" name="ubuntu" dev="sda1" ino=2387 scontext=system_u:system_r:cachefilesd_t:s0 tcontext=unconfined_u:object_r:user_home_dir_t:s0 tclass=dir permissive=1', 'type=AVC msg=audit(1612634408.842:8469): avc: denied { create } for pid=15643 comm="cachefilesd" name="1612634407.15643.core" scontext=system_u:system_r:cachefilesd_t:s0 tcontext=system_u:object_r:user_home_t:s0 tclass=file permissive=1', 'type=AVC msg=audit(1612634408.842:8467): avc: denied { write } for pid=15643 comm="cachefilesd" name="coredump" dev="sda1" ino=524299 scontext=system_u:system_r:cachefilesd_t:s0 tcontext=unconfined_u:object_r:user_home_t:s0 tclass=dir permissive=1'] 1 jobs: ['5861442'] suites: ['clusters/1a5s-mds-1c-client-3node', 'conf/{client', 'distro/{rhel_8}', 'fs/workload/{begin', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{frag_enable', 'ranks/3', 'rhel_8}', 'scrub/yes', 'session_timeout', 'tasks/{0-check-counter', 'whitelist_health', 'whitelist_wrongly_marked_down}', 'workunit/suites/iogen}}'] Crash: Command failed on gibba004 with status 1: 'sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp' ceph version 17.0.0-29-g70828cb1 (70828cb1a8014f8c92cf86014114fd7aab6861c9) pacific (dev) 1: /lib64/libpthread.so.0(+0x12b20) [0x7f47b00ceb20] 2: gsignal() 3: abort() 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a9) [0x7f47b1321a6b] 5: /usr/lib64/ceph/libceph-common.so.2(+0x275c34) [0x7f47b1321c34] 6: (ScrubStack::handle_scrub(boost::intrusive_ptr const&)+0x1412) [0x55aadba8ed82] 7: (ScrubStack::dispatch(boost::intrusive_ptr const&)+0x134) [0x55aadba8f004] 8: (MDSRank::handle_message(boost::intrusive_ptr const&)+0xc22) [0x55aadb7bfa72] 9: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x7bb) [0x55aadb7c207b] 10: (MDSRank::retry_dispatch(boost::intrusive_ptr const&)+0x16) [0x55aadb7c2616] 11: (MDSContext::complete(int)+0x56) [0x55aadbac1aa6] 12: (C_GatherBase::sub_finish(MDSContext*, int)+0x10b) [0x55aadb7f455b] 13: (C_GatherBase::C_GatherSub::complete(int)+0x25) [0x55aadb7f4995] 14: (MDSRank::_advance_queues()+0x84) [0x55aadb7c10e4] 15: (MDSRank::ProgressThread::entry()+0xc5) [0x55aadb7c1805] 16: /lib64/libpthread.so.0(+0x814a) [0x7f47b00c414a] 17: clone() 1 jobs: ['5861445'] suites: ['clusters/1a5s-mds-1c-client-3node', 'conf/{client', 'distro/{centos_8}', 'fs/workload/{begin', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-bitmap', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{frag_enable', 'ranks/5', 'rhel_8}', 'scrub/yes', 'session_timeout', 'tasks/{0-check-counter', 'whitelist_health', 'whitelist_wrongly_marked_down}', 'workunit/fs/misc}}']