Found 170 jobs Missing teuthology log /home/teuthworker/archive/yuriw-2022-11-23_16:39:25-fs-wip-yuri2-testing-2022-10-19-1252-pacific-distro-default-smithi/7087891/teuthology.log Found 46 distinct failure reasons Failure: Test failure: test_flush (tasks.cephfs.test_readahead.TestReadahead) 1 jobs: ['7087904'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/client-readahead}'] Failure: Test failure: test_replay_create (tasks.cephfs.test_cap_flush.TestCapFlush) 1 jobs: ['7087898'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{ignorelist_health', 'tasks/cap-flush}'] Failure: Command failed (workunit test fs/snaps/snaptest-git-ceph.sh) on smithi049 with status 128: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=8971b5542afa0f81acdcd730197df1f921e9bf83 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/snaps/snaptest-git-ceph.sh' 10 jobs: ['7087771', '7087918', '7087812', '7087832', '7087783', '7087788', '7087852', '7087900', '7087858', '7087884'] suites intersection: ['conf/{client', 'mds', 'mon', 'osd}'] suites union: ['2-workunit/cfuse_workunit_snaptests}}', '2-workunit/fs/snaps}}', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs/snaps/{begin', 'fs/thrash/multifs/{begin', 'fs/thrash/workloads/{begin', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'k-testing}', 'mds', 'mon', 'mount/fuse', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{frag', 'overrides/{ignorelist_health', 'prefetch_dirfrags/no', 'races', 'ranks/1', 'rhel_8}', 'session_timeout', 'tasks/workunit/snaps}', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'thrashosds-health}'] Dead: None 2 jobs: ['7087891', '7087915'] Failure: Test failure: test_cap_acquisition_throttle_readdir (tasks.cephfs.test_client_limits.TestClientLimits) 1 jobs: ['7087902'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/client-limits}'] Failure: Test failure: test_clone_failure_status_failed (tasks.cephfs.test_volumes.TestSubvolumeSnapshotClones) 2 jobs: ['7087772', '7087901'] suites intersection: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'fs/volumes/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/volumes/{overrides', 'test/clone}}'] suites union: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'fs/volumes/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/volumes/{overrides', 'test/clone}}'] Failure: Test failure: test_apply_tag (tasks.cephfs.test_forward_scrub.TestForwardScrub) 2 jobs: ['7087927', '7087776'] suites intersection: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/forward-scrub}'] suites union: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/fuse', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{ignorelist_health', 'tasks/forward-scrub}'] Failure: Test failure: test_volume_create (tasks.cephfs.test_volumes.TestVolumes) 1 jobs: ['7087876'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/volumes/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/volumes/{overrides', 'test/basic}}'] Failure: Test failure: test_newops_getvxattr (tasks.cephfs.test_newops.TestNewOps) 1 jobs: ['7087882'] suites: ['1-client-upgrade', '2-client-sanity}}', 'centos_latest', 'clusters/{1-mds-1-client-micro}', 'conf/{client', 'fs/upgrade/upgraded_client/from_nautilus/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/{0-nautilus'] Failure: Test failure: test_add_ancestor_and_child_directory (tasks.cephfs.test_mirroring.TestMirroring) 2 jobs: ['7087845', '7087856'] suites intersection: ['mount/fuse'] suites union: ['centos_latest', 'cephfs-mirror/one-per-cluster', 'clients/mirror', 'clients/{mirror}', 'cluster/1-node', 'cluster/{1-node}', 'debug', 'fs/mirror/{begin', 'fs/valgrind/{begin', 'mirror/{cephfs-mirror/one-per-cluster', 'mount/fuse', 'objectstore/bluestore-bitmap', 'overrides/whitelist_health', 'overrides/{whitelist_health}', 'supported-random-distros$/{centos_8}', 'tasks/mirror}', 'tasks/mirror}}'] Failure: Test failure: test_max_items_per_obj (tasks.cephfs.test_openfiletable.OpenFileTable) 1 jobs: ['7087809'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/openfiletable}'] Failure: Test failure: test_mount_all_caps_absent (tasks.cephfs.test_multifs_auth.TestClientsWithoutAuth) 3 jobs: ['7087849', '7087811', '7087913'] suites intersection: ['clusters/1a3s-mds-2c-client', 'conf/{client', 'fs/multifs/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'osd}', 'overrides/{ignorelist_health', 'tasks/multifs-auth}'] suites union: ['clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'fs/multifs/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mon-debug}', 'mount/fuse', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{ignorelist_health', 'tasks/multifs-auth}'] Failure: Test failure: test_deep_split (tasks.cephfs.test_fragment.TestFragmentation) 2 jobs: ['7087929', '7087778'] suites intersection: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/fragment}'] suites union: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/fragment}'] Failure: Test failure: test_binary_metafile_on_legacy_to_v1_upgrade (tasks.cephfs.test_volumes.TestMisc) 2 jobs: ['7087795', '7087925'] suites intersection: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'fs/volumes/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/volumes/{overrides', 'test/misc}}'] suites union: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs/volumes/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{ignorelist_health', 'tasks/volumes/{overrides', 'test/misc}}'] Failure: Test failure: test_acls (tasks.cephfs.test_acls.TestACLs) 1 jobs: ['7087871'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/acls}'] Failure: Test failure: test_recover_fs_after_fsmap_removal (tasks.cephfs.test_recovery_fs.TestFSRecovery) 1 jobs: ['7087824'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{ignorelist_health', 'tasks/recovery-fs}'] Failure: Test failure: test_client_metrics_and_metadata (tasks.cephfs.test_mds_metrics.TestMDSMetrics) 1 jobs: ['7087801'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{ignorelist_health', 'tasks/metrics}'] Failure: Test failure: test_scrub_checks (tasks.cephfs.test_scrub_checks.TestScrubChecks) 1 jobs: ['7087828'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/scrub}'] Failure: Test failure: test_apply_tag (tasks.cephfs.test_multimds_misc.TestScrub2) 1 jobs: ['7087805'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{ignorelist_health', 'tasks/multimds_misc}'] Failure: Test failure: test_ephemeral_pin_dist_conf_off (tasks.cephfs.test_exports.TestEphemeralPins) 2 jobs: ['7087773', '7087923'] suites intersection: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/exports}'] suites union: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/exports}'] Failure: Test failure: test_barrier (tasks.cephfs.test_full.TestClusterFull) 1 jobs: ['7087794'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/mds-full}'] Failure: Test failure: test_inject_to_empty (tasks.cephfs.test_journal_repair.TestJournalRepair) 2 jobs: ['7087937', '7087786'] suites intersection: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/journal-repair}'] suites union: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/fuse', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{ignorelist_health', 'tasks/journal-repair}'] Failure: Test failure: test_damaged_dentry (tasks.cephfs.test_damage.TestDamage) 1 jobs: ['7087917'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{ignorelist_health', 'tasks/damage}'] Failure: Test failure: test_backtrace_repair (tasks.cephfs.test_auto_repair.TestMDSAutoRepair) 1 jobs: ['7087889'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/auto-repair}'] Failure: Test failure: test_rebuild_simple (tasks.cephfs.test_recovery_pool.TestRecoveryPool) 1 jobs: ['7087881'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{ignorelist_health', 'tasks/alternate-pool}'] Failure: Test failure: test_cd_with_args (tasks.cephfs.test_cephfs_shell.TestCD) 1 jobs: ['7087851'] suites: ['clusters/1-mds-1-client-coloc', 'conf/{client', 'distro/{rhel_8}', 'fs/shell/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/cephfs-shell}'] Failure: Test failure: test_fstop (tasks.cephfs.test_fstop.TestFSTop) 1 jobs: ['7087853'] suites: ['cluster/{1-node}', 'fs/top/{begin', 'mount/fuse', 'objectstore/bluestore-bitmap', 'overrides/ignorelist_health', 'supported-random-distros$/{ubuntu_latest}', 'tasks/fstop}'] Failure: Test failure: test_fragmented_injection (tasks.cephfs.test_data_scan.TestDataScan) 2 jobs: ['7087919', '7087770'] suites intersection: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/data-scan}'] suites union: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/fuse', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{ignorelist_health', 'tasks/data-scan}'] Failure: Test failure: test_drop_cache_command (tasks.cephfs.test_misc.TestCacheDrop) 2 jobs: ['7087934', '7087848'] suites intersection: ['conf/{client', 'distros/ubuntu_latest', 'fs/multiclient/{begin', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'osd}', 'overrides/{ignorelist_health', 'tasks/cephfs_misc_tests}'] suites union: ['clusters/1-mds-2-client', 'clusters/1-mds-3-client', 'conf/{client', 'distros/ubuntu_latest', 'fs/multiclient/{begin', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/cephfs_misc_tests}'] Failure: Test failure: test_backtrace (tasks.cephfs.test_backtrace.TestBacktrace) 1 jobs: ['7087896'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{ignorelist_health', 'tasks/backtrace}'] Timeout 6h running clone.client.0/qa/workunits/suites/fsstress.sh 2 jobs: ['7087793', '7087816'] suites intersection: ['clusters/1a5s-mds-1c-client', 'conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'ranks/5'] suites union: ['2-workunit/suites/fsstress}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{ubuntu_latest}', 'fs/thrash/workloads/{begin', 'fs/verify/{begin', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/fuse', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{frag', 'overrides/{ignorelist_health', 'prefetch_dirfrags/no', 'races', 'ranks/5', 'session_timeout', 'session_timeout}', 'tasks/fsstress', 'tasks/{1-thrash/mon', 'thrashosds-health}', 'validater/lockdep}'] Failure: Test failure: test_15303 (tasks.cephfs.test_volume_client.TestVolumeClient) 1 jobs: ['7087857'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/volumes/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/volume-client}'] Failure: Test failure: test_join_fs_drop (tasks.cephfs.test_failover.TestClusterAffinity) 2 jobs: ['7087883', '7087781'] suites intersection: ['clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/multifs/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'osd}', 'overrides/{ignorelist_health', 'tasks/failover}'] suites union: ['clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/multifs/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mon-debug}', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/failover}'] Failure: Test failure: test_abort_conn (tasks.cephfs.test_client_recovery.TestClientRecovery) 1 jobs: ['7087909'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{ignorelist_health', 'tasks/client-recovery}'] Failure: Test failure: test_dump_subtrees (tasks.cephfs.test_admin.TestAdminCommandDumpTree) 1 jobs: ['7087875'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{ignorelist_health', 'tasks/admin}'] Failure: Test failure: test_concurrent_snap_creates (tasks.cephfs.test_snap_schedules.TestSnapSchedules) 1 jobs: ['7087834'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{ignorelist_health', 'tasks/snap-schedule}'] Failure: Test failure: test_mount_conn_close (tasks.cephfs.test_sessionmap.TestSessionMap) 1 jobs: ['7087831'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/sessionmap}'] Failure: Test failure: test_flush (tasks.cephfs.test_flush.TestFlush) 1 jobs: ['7087790'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{ignorelist_health', 'tasks/mds-flush}'] Failure: Command failed (workunit test suites/fsx.sh) on smithi008 with status 128: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=8971b5542afa0f81acdcd730197df1f921e9bf83 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsx.sh' 1 jobs: ['7087911'] suites: ['clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'distro/{centos_8}', 'fs/workload/{begin', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{frag', 'ranks/1', 'rhel_8}', 'scrub/no', 'session_timeout}', 'standby-replay', 'tasks/{0-check-counter', 'workunit/suites/fsx}', 'wsync/{yes}}'] Failure: Test failure: test_allow_new_snaps_config (tasks.cephfs.test_snapshots.TestSnapshots) 1 jobs: ['7087836'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{ignorelist_health', 'tasks/snapshots}'] Failure: Test failure: test_basic (tasks.cephfs.test_dump_tree.TestDumpTree) 1 jobs: ['7087886'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/asok_dump_tree}'] Failure: Test failure: test_subvolume_group_snapshot_unsupported_status (tasks.cephfs.test_volumes.TestSubvolumeGroupSnapshots) 1 jobs: ['7087822'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/volumes/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{ignorelist_health', 'tasks/volumes/{overrides', 'test/snapshot}}'] Failure: Test failure: test_remote_update_df (tasks.cephfs.test_quota.TestQuota) 1 jobs: ['7087820'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{ignorelist_health', 'tasks/quota}'] Failure: Test failure: test_forbidden_modification (tasks.cephfs.test_pool_perm.TestPoolPerm) 1 jobs: ['7087814'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/pool-perm}'] Failure: Test failure: test_journal_migration (tasks.cephfs.test_journal_migration.TestJournalMigration) 1 jobs: ['7087859'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/test_journal_migration}'] Failure: Test failure: test_dir_deletion (tasks.cephfs.test_strays.TestStrays) 1 jobs: ['7087839'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/strays}']