Found 367 jobs Misunderstood line: Binary file /home/teuthworker/archive/xiubli-2020-09-21_06:42:37-fs-wip-lxb-defer-encoding-20200921-1157-distro-basic-smithi/5455861/teuthology.log matches Misunderstood line: Found 9 distinct failure reasons Failure: Command failed on smithi192 with status 1: 'sudo yum -y install ceph-radosgw ceph-test ceph ceph-fuse cephfs-java libcephfs_jni1 libcephfs1 librados2 librbd1 python-ceph rbd-fuse python3-cephfs bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel' 12 jobs: ['5456127', '5456125', '5456112', '5456122', '5456126', '5456124', '5456133', '5456121', '5456129', '5456120', '5456108', '5456132'] suites intersection: ['conf/{client.yaml', 'mds.yaml', 'mon.yaml', 'mount/fuse.yaml', 'osd.yaml}', 'whitelist_health.yaml', 'whitelist_wrongly_marked_down.yaml}'] suites union: ['centos_latest.yaml', 'ceph-thrash/mds.yaml', 'ceph-thrash/mon.yaml', 'clusters/1-mds-1-client-coloc.yaml', 'clusters/fixed-2-ucephfs.yaml', 'conf/{client.yaml', 'frag_enable.yaml', 'fs/32bits/{begin.yaml', 'fs/basic_workload/{begin.yaml', 'fs/thrash/{begin.yaml', 'fs/traceless/{begin.yaml', 'fs/verify/{begin.yaml', 'inline/no.yaml', 'inline/yes.yaml', 'mds.yaml', 'mon-debug.yaml', 'mon.yaml', 'mount/fuse.yaml', 'msgr-failures/none.yaml', 'msgr-failures/osd-mds-delay.yaml', 'objectstore-ec/bluestore-bitmap.yaml', 'objectstore-ec/bluestore-comp-ec-root.yaml', 'objectstore-ec/bluestore-comp.yaml', 'objectstore-ec/bluestore-ec-root.yaml', 'objectstore-ec/filestore-xfs.yaml', 'omap_limit/10.yaml', 'omap_limit/10000.yaml', 'osd.yaml}', 'overrides/{faked-ino.yaml', 'overrides/{frag_enable.yaml', 'supported-random-distros$/{centos_8.yaml}', 'supported-random-distros$/{rhel_8.yaml}', 'tasks/cfuse_workunit_misc.yaml}', 'tasks/cfuse_workunit_norstats.yaml}', 'tasks/cfuse_workunit_suites_blogbench.yaml', 'tasks/cfuse_workunit_suites_blogbench.yaml}', 'tasks/cfuse_workunit_suites_dbench.yaml}', 'tasks/cfuse_workunit_suites_ffsb.yaml}', 'tasks/cfuse_workunit_suites_fsstress.yaml', 'tasks/cfuse_workunit_suites_fsstress.yaml}', 'tasks/cfuse_workunit_suites_fsync.yaml}', 'tasks/cfuse_workunit_suites_pjd.yaml}', 'tasks/cfuse_workunit_trivial_sync.yaml}', 'traceless/50pc.yaml}', 'validater/lockdep.yaml}', 'whitelist_health.yaml', 'whitelist_wrongly_marked_down.yaml}'] Failure: Command failed on smithi101 with status 1: 'sudo logrotate /etc/logrotate.d/ceph-test.conf' 1 jobs: ['5456009'] suites: ['clusters/fixed-2-ucephfs.yaml', 'conf/{client.yaml', 'fs/basic_workload/{begin.yaml', 'inline/yes.yaml', 'mds.yaml', 'mon.yaml', 'mount/fuse.yaml', 'objectstore-ec/bluestore-comp-ec-root.yaml', 'omap_limit/10.yaml', 'osd.yaml}', 'overrides/{frag_enable.yaml', 'supported-random-distros$/{ubuntu_latest.yaml}', 'tasks/cfuse_workunit_norstats.yaml}', 'whitelist_health.yaml', 'whitelist_wrongly_marked_down.yaml}'] Failure: Command failed on smithi050 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.0.0-5757-g7ebc9808-1bionic ceph-mds=16.0.0-5757-g7ebc9808-1bionic ceph-common=16.0.0-5757-g7ebc9808-1bionic ceph-fuse=16.0.0-5757-g7ebc9808-1bionic ceph-test=16.0.0-5757-g7ebc9808-1bionic radosgw=16.0.0-5757-g7ebc9808-1bionic python-ceph=16.0.0-5757-g7ebc9808-1bionic libcephfs1=16.0.0-5757-g7ebc9808-1bionic libcephfs-java=16.0.0-5757-g7ebc9808-1bionic libcephfs-jni=16.0.0-5757-g7ebc9808-1bionic librados2=16.0.0-5757-g7ebc9808-1bionic librbd1=16.0.0-5757-g7ebc9808-1bionic rbd-fuse=16.0.0-5757-g7ebc9808-1bionic python3-cephfs=16.0.0-5757-g7ebc9808-1bionic cephfs-shell=16.0.0-5757-g7ebc9808-1bionic' 10 jobs: ['5456109', '5456128', '5456105', '5456131', '5456104', '5456130', '5456117', '5456111', '5456115', '5456123'] suites intersection: ['conf/{client.yaml', 'mds.yaml', 'mon.yaml', 'mount/fuse.yaml', 'osd.yaml}', 'overrides/{frag_enable.yaml', 'whitelist_health.yaml', 'whitelist_wrongly_marked_down.yaml}'] suites union: ['clusters/1-mds-2-client.yaml', 'clusters/1-mds-4-client-coloc.yaml', 'clusters/fixed-2-ucephfs.yaml', 'conf/{client.yaml', 'distros/ubuntu_latest.yaml', 'fs/basic_functional/{begin.yaml', 'fs/basic_workload/{begin.yaml', 'fs/multiclient/{begin.yaml', 'fs/permission/{begin.yaml', 'inline/no.yaml', 'inline/yes.yaml', 'mds.yaml', 'mon.yaml', 'mount/fuse.yaml', 'no_client_pidfile.yaml', 'objectstore-ec/bluestore-bitmap.yaml', 'objectstore-ec/bluestore-comp-ec-root.yaml', 'objectstore-ec/bluestore-ec-root.yaml', 'objectstore-ec/filestore-xfs.yaml', 'objectstore/bluestore-bitmap.yaml', 'omap_limit/10.yaml', 'omap_limit/10000.yaml', 'osd.yaml}', 'overrides/{frag_enable.yaml', 'supported-random-distros$/{ubuntu_latest.yaml}', 'tasks/backtrace.yaml}', 'tasks/cephfs-shell.yaml}', 'tasks/cephfs_misc_tests.yaml}', 'tasks/cfuse_workunit_misc.yaml}', 'tasks/cfuse_workunit_misc_test_o_trunc.yaml}', 'tasks/cfuse_workunit_suites_fsstress.yaml}', 'tasks/cfuse_workunit_suites_fsx.yaml}', 'tasks/cfuse_workunit_suites_iogen.yaml}', 'tasks/cfuse_workunit_suites_truncate_delay.yaml}', 'tasks/libcephfs_interface_tests.yaml}', 'whitelist_health.yaml', 'whitelist_wrongly_marked_down.yaml}'] Failure: No module named 'tasks.ceph' 5 jobs: ['5456119', '5456116', '5456113', '5456118', '5456103'] suites intersection: ['conf/{client.yaml', 'mds.yaml', 'mon.yaml', 'mount/fuse.yaml', 'osd.yaml}', 'overrides/{frag_enable.yaml', 'whitelist_health.yaml', 'whitelist_wrongly_marked_down.yaml}'] suites union: ['ceph-thrash/mon.yaml', 'clusters/1-mds-1-client-coloc.yaml', 'clusters/1-mds-4-client-coloc.yaml', 'clusters/1a3s-mds-2c-client.yaml', 'clusters/fixed-2-ucephfs.yaml', 'conf/{client.yaml', 'fs/basic_functional/{begin.yaml', 'fs/basic_workload/{begin.yaml', 'fs/multifs/{begin.yaml', 'fs/snaps/{begin.yaml', 'fs/thrash/{begin.yaml', 'inline/no.yaml', 'mds.yaml', 'mon-debug.yaml', 'mon.yaml', 'mount/fuse.yaml', 'msgr-failures/none.yaml', 'no_client_pidfile.yaml', 'objectstore-ec/bluestore-bitmap.yaml', 'objectstore-ec/bluestore-comp-ec-root.yaml', 'objectstore-ec/bluestore-ec-root.yaml', 'objectstore/bluestore-ec-root.yaml', 'omap_limit/10000.yaml', 'osd.yaml}', 'overrides/{frag_enable.yaml', 'supported-random-distros$/{centos_8.yaml}', 'supported-random-distros$/{rhel_8.yaml}', 'supported-random-distros$/{ubuntu_latest.yaml}', 'tasks/cap-flush.yaml}', 'tasks/cfuse_workunit_snaptests.yaml}', 'tasks/cfuse_workunit_suites_fsync.yaml}', 'tasks/failover.yaml}', 'tasks/snaptests.yaml}', 'whitelist_health.yaml', 'whitelist_wrongly_marked_down.yaml}'] Failure: Could not find an image for centos 8 5 jobs: ['5456019', '5455770', '5455935', '5455777', '5456099'] Failure: "2020-09-25T03:09:16.529771+0000 mon.a (mon.0) 55 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN)" in cluster log 1 jobs: ['5455842'] suites: ['clusters/fixed-2-ucephfs.yaml', 'conf/{client.yaml', 'fs/basic_workload/{begin.yaml', 'inline/no.yaml', 'mds.yaml', 'mon.yaml', 'mount/fuse.yaml', 'objectstore-ec/bluestore-ec-root.yaml', 'omap_limit/10.yaml', 'osd.yaml}', 'overrides/{frag_enable.yaml', 'supported-random-distros$/{ubuntu_latest.yaml}', 'tasks/cfuse_workunit_suites_ffsb.yaml}', 'whitelist_health.yaml', 'whitelist_wrongly_marked_down.yaml}'] Failure: Command failed on smithi039 with status 128: 'rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone git://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 7ebc9808d99a9037db1aace7340eda9c4e98cd94' 1 jobs: ['5456107'] suites: ['clusters/fixed-2-ucephfs.yaml', 'conf/{client.yaml', 'fs/basic_workload/{begin.yaml', 'inline/yes.yaml', 'mds.yaml', 'mon.yaml', 'mount/fuse.yaml', 'objectstore-ec/bluestore-ec-root.yaml', 'omap_limit/10.yaml', 'osd.yaml}', 'overrides/{frag_enable.yaml', 'supported-random-distros$/{centos_8.yaml}', 'tasks/cfuse_workunit_suites_pjd.yaml}', 'whitelist_health.yaml', 'whitelist_wrongly_marked_down.yaml}'] Failure: Command failed (workunit test fs/quota/quota.sh) on smithi174 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.3/client.3/tmp && cd -- /home/ubuntu/cephtest/mnt.3/client.3/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=7ebc9808d99a9037db1aace7340eda9c4e98cd94 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="3" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.3 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.3/qa/workunits/fs/quota/quota.sh' 1 jobs: ['5455857'] suites: ['clusters/1-mds-4-client-coloc.yaml', 'conf/{client.yaml', 'fs/basic_functional/{begin.yaml', 'mds.yaml', 'mon.yaml', 'mount/fuse.yaml', 'no_client_pidfile.yaml', 'objectstore/bluestore-ec-root.yaml', 'osd.yaml}', 'overrides/{frag_enable.yaml', 'supported-random-distros$/{ubuntu_latest.yaml}', 'tasks/cfuse_workunit_quota.yaml}', 'whitelist_health.yaml', 'whitelist_wrongly_marked_down.yaml}'] Valgrind: 2 jobs: ['5455861', '5456029'] suites intersection: ['conf/{client.yaml', 'mds.yaml', 'mon.yaml', 'mount/fuse.yaml', 'osd.yaml}', 'overrides/{frag_enable.yaml', 'whitelist_health.yaml', 'whitelist_wrongly_marked_down.yaml}'] suites union: ['centos_latest.yaml', 'clusters/1-mds-4-client-coloc.yaml', 'clusters/fixed-2-ucephfs.yaml', 'conf/{client.yaml', 'fs/basic_functional/{begin.yaml', 'fs/verify/{begin.yaml', 'mds.yaml', 'mon-debug.yaml', 'mon.yaml', 'mount/fuse.yaml', 'no_client_pidfile.yaml', 'objectstore-ec/bluestore-ec-root.yaml', 'objectstore/bluestore-ec-root.yaml', 'osd.yaml}', 'overrides/{frag_enable.yaml', 'supported-random-distros$/{ubuntu_latest.yaml}', 'tasks/cfuse_workunit_suites_fsstress.yaml', 'tasks/test_journal_migration.yaml}', 'validater/valgrind.yaml}', 'whitelist_health.yaml', 'whitelist_wrongly_marked_down.yaml}']