Found 156 jobs Found 22 distinct failure reasons Failure: Command failed on smithi042 with status 22: "sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b583906-b90d-11ed-9ae9-001a4aab830c -e sha1=802de851643c97960b33dacc1e3ce8486e6d7d3c -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr'" 6 jobs: ['7184141', '7184113', '7184170', '7184052', '7184192', '7184086'] suites intersection: ['1-volume/{0-create', '2-client', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/v16.2.4'] suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client', '3-inline/no', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/no', 'fail_fs/yes', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/v16.2.4'] Failure: {'smithi186.front.sepia.ceph.com': {'censored': "the output has been hidden due to the fact that 'no_log: true' was specified for this result", 'changed': True}} 1 jobs: ['7184162'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/fsx}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'objectstore-ec/bluestore-bitmap', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'ranks/1', 'rhel_8}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'wsync/yes}'] Failure: Command failed (workunit test suites/fsstress.sh) on smithi042 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=802de851643c97960b33dacc1e3ce8486e6d7d3c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh' 2 jobs: ['7184073', '7184180'] suites intersection: ['conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}'] suites union: ['1-ceph', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '2-logrotate}', '2-workunit/suites/fsstress}}', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{rhel_8}', 'fail_fs/yes', 'fs/thrash/workloads/{begin/{0-install', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-bitmap', 'osd}', 'overrides/{frag', 'overrides/{ignorelist_health', 'pg-warn', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'races', 'ranks/5', 'roles', 'session_timeout', 'syntax}', 'tasks/{0-from/pacific', 'tasks/{1-thrash/osd', 'thrashosds-health}'] Failure: Command failed on smithi186 with status 1: 'TESTDIR=/home/ubuntu/cephtest bash -s' 1 jobs: ['7184164'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/no', '4-flush/no', '5-workunit/suites/fsync-tester}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/3', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'wsync/no}'] Failure: error during scrub thrashing: rank damage found: {'backtrace'} 3 jobs: ['7184119', '7184182', '7184142'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'ranks/multi/{export-check', 'session_timeout}', 'standby-replay'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', '5-workunit/direct_io}}', '5-workunit/fs/misc}}', '5-workunit/fs/norstats}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'rhel_8}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-no-extra-options}', 'wsync/yes}'] Crash: Command failed (workunit test suites/fsstress.sh) on smithi017 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=802de851643c97960b33dacc1e3ce8486e6d7d3c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh' ceph version 18.0.0-2462-g802de851 (802de851643c97960b33dacc1e3ce8486e6d7d3c) reef (dev) 1: /lib64/libpthread.so.0(+0x12cf0) [0x7fb934c12cf0] 2: gsignal() 3: abort() 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x18f) [0x7fb9368a7e15] 5: /usr/lib64/ceph/libceph-common.so.2(+0x2a7f81) [0x7fb9368a7f81] 6: ceph-mds(+0x2c2263) [0x559efed6b263] 7: (Server::handle_peer_rename_prep(boost::intrusive_ptr&)+0x1ccf) [0x559efed4e5df] 8: (MDSContext::complete(int)+0x5f) [0x559efefe742f] 9: (MDSRank::_advance_queues()+0xaa) [0x559efec6c13a] 10: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x1ce) [0x559efec6cc2e] 11: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x559efec6d61c] 12: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x1bf) [0x559efec5795f] 13: (Messenger::ms_deliver_dispatch(boost::intrusive_ptr const&)+0x478) [0x7fb936b1d398] 14: (DispatchQueue::entry()+0x50f) [0x7fb936b1a53f] 15: (DispatchQueue::DispatchThread::entry()+0x11) [0x7fb936bdf281] 16: /lib64/libpthread.so.0(+0x81ca) [0x7fb934c081ca] 17: clone() 5 jobs: ['7184124', '7184176', '7184089', '7184187', '7184098'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}'] suites union: ['1-ceph', '2-logrotate}', '2-workunit/suites/fsstress}}', '2-workunit/suites/pjd}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'fs/thrash/workloads/{begin/{0-install', 'fs/verify/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/fuse', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{frag', 'overrides/{ignorelist_health', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/3', 'ranks/5', 'session_timeout', 'session_timeout}', 'tasks/dbench', 'tasks/fsstress', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'thrashosds-health}', 'validater/lockdep}', 'validater/valgrind}'] Failure: Command failed (workunit test suites/pjd.sh) on smithi089 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=802de851643c97960b33dacc1e3ce8486e6d7d3c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh' 2 jobs: ['7184178', '7184110'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/no', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'session_timeout}', 'standby-replay'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/no', '4-flush/no', '4-flush/yes', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/3', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/1', 'ranks/multi/{export-check', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}'] Dead: b'2023-03-03T02:31:38.834 DEBUG:teuthology.exit:Finished running handlers' 2 jobs: ['7184129', '7184191'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'ranks/multi/{export-check', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{no-subvolume}'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', '5-workunit/kernel_untar_build}}', '5-workunit/postgres}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'rhel_8}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{no-subvolume}', 'wsync/yes}'] Failure: {'smithi186.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'failures': ['No package qemu-kvm-block-rbd available.'], 'invocation': {'module_args': {'allow_downgrade': False, 'allowerasing': False, 'autoremove': False, 'bugfix': False, 'conf_file': None, 'disable_excludes': None, 'disable_gpg_check': False, 'disable_plugin': [], 'disablerepo': [], 'download_dir': None, 'download_only': False, 'enable_plugin': [], 'enablerepo': [], 'exclude': [], 'install_repoquery': True, 'install_weak_deps': True, 'installroot': '/', 'list': None, 'lock_timeout': 30, 'name': ['dnf-utils', 'git-all', 'sysstat', 'libedit', 'boost-thread', 'xfsprogs', 'gdisk', 'parted', 'libgcrypt', 'fuse-libs', 'openssl', 'libuuid', 'attr', 'ant', 'lsof', 'gettext', 'bc', 'xfsdump', 'blktrace', 'usbredir', 'podman', 'redhat-lsb', 'firewalld', 'wget', 'libev-devel', 'valgrind', 'nfs-utils', 'ncurses-devel', 'gcc', 'git', 'make', 'python3-nose', 'python3-virtualenv', 'genisoimage', 'qemu-img', 'qemu-kvm-core', 'qemu-kvm-block-rbd', 'libacl-devel', 'autoconf', 'gdb', 'iozone', 'lvm2'], 'releasever': None, 'security': False, 'skip_broken': False, 'state': 'present', 'update_cache': False, 'update_only': False, 'validate_certs': True}}, 'msg': 'Failed to install some of the specified packages', 'rc': 1, 'results': []}} 1 jobs: ['7184204'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/5', 'objectstore-ec/bluestore-bitmap', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'ranks/multi/{export-check', 'replication/default}', 'rhel_8}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated}', 'wsync/yes}'] Failure: Test failure: test_metrics_mds_grow_and_shrink (tasks.cephfs.test_mds_metrics.TestMDSMetrics) 1 jobs: ['7184118'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/metrics}'] Timeout 120 running ceph --cluster ceph tell mds.1:0 scrub status 1 jobs: ['7184053'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/no', '5-workunit/fs/misc}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/5', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'wsync/no}'] Failure: Test failure: test_cephfs_mirror_restart_sync_on_blocklist (tasks.cephfs.test_mirroring.TestMirroring) 1 jobs: ['7184153'] suites: ['1-ceph', '2-logrotate}', 'centos_latest', 'clients/mirror', 'cluster/1-node', 'debug', 'fs/valgrind/{begin/{0-install', 'mirror/{cephfs-mirror/one-per-cluster', 'mount/fuse', 'overrides/whitelist_health', 'tasks/mirror}}'] Failure: Test failure: test_migrate_unlinked_dir (tasks.cephfs.test_strays.TestStrays) 1 jobs: ['7184166'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/strays}'] Failure: Test failure: test_subvolumegroup_pin_distributed (tasks.cephfs.test_volumes.TestSubvolumeGroups), test_subvolumegroup_pin_distributed (tasks.cephfs.test_volumes.TestSubvolumeGroups) 2 jobs: ['7184094', '7184201'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'fs/volumes/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/volumes/{overrides', 'test/basic}}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'fs/volumes/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/volumes/{overrides', 'test/basic}}'] Failure: SSH connection to smithi186 was lost: 'sudo rpm -ivh --oldpackage --replacefiles --replacepkgs /tmp/kernel.x86_64.rpm' 1 jobs: ['7184179'] suites: ['1-ceph', '2-logrotate}', 'clusters/fixed-2-ucephfs', 'conf/{client', 'distro/{rhel_8}', 'fs/traceless/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{frag', 'tasks/cfuse_workunit_suites_ffsb', 'traceless/50pc}'] Failure: Test failure: test_down (tasks.cephfs.test_failover.TestClusterResize) 2 jobs: ['7184150', '7184092'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/multifs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'osd}', 'overrides/{ignorelist_health', 'tasks/failover}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/multifs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/failover}'] Failure: {'smithi186.front.sepia.ceph.com': {'_ansible_no_log': False, 'msg': 'Failed to connect to the host via ssh: ssh: connect to host smithi186.front.sepia.ceph.com port 22: No route to host'}} 1 jobs: ['7184197'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/blogbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-comp', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'ranks/1', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated}'] Failure: Test failure: test_rebuild_simple (tasks.cephfs.test_recovery_pool.TestRecoveryPool) 1 jobs: ['7184190'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/alternate-pool}'] Failure: Command failed (workunit test suites/dbench.sh) on smithi005 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=802de851643c97960b33dacc1e3ce8486e6d7d3c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/dbench.sh' 3 jobs: ['7184071', '7184135', '7184202'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '5-workunit/suites/dbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'ranks/multi/{export-check', 'session_timeout}', 'standby-replay'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/suites/dbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/legacy', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'rhel_8}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{no-subvolume}', 'tasks/{0-subvolume/{with-no-extra-options}', 'wsync/no}', 'wsync/yes}'] Failure: Cannot connect to remote host smithi186 2 jobs: ['7184184', '7184186'] suites intersection: ['conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}'] suites union: ['1-cephadm', '1-check-counter', '1-client', '2-logrotate}', '2-scrub/no', '2-upgrade', '3-compat_client/quincy}}', '3-snaps/no', '4-flush/no', '5-workunit/fs/misc}}', 'begin/{0-install', 'centos_latest', 'clusters/1-mds-2-client-micro', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'multimds/yes', 'n/5', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn}', 'ranks/multi/{export-check', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-octopus', 'tasks/{0-subvolume/{with-namespace-isolated}', 'wsync/no}'] Crash: Test failure: test_inject_to_empty (tasks.cephfs.test_journal_repair.TestJournalRepair) ceph version 18.0.0-2462-g802de851 (802de851643c97960b33dacc1e3ce8486e6d7d3c) reef (dev) 1: /lib64/libpthread.so.0(+0x12ce0) [0x7eff04de6ce0] 2: gsignal() 3: abort() 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x18f) [0x7eff06031e15] 5: /usr/lib64/ceph/libceph-common.so.2(+0x2a7f81) [0x7eff06031f81] 6: (CDir::log_mark_dirty()+0xfa) [0x563d52a6e3ea] 7: (CDir::_omap_fetched(ceph::buffer::v15_2_0::list&, std::map, std::allocator >, ceph::buffer::v15_2_0::list, std::less, std::allocator > >, std::allocator, std::allocator > const, ceph::buffer::v15_2_0::list> > >&, bool, std::set, std::allocator >, std::less, std::allocator > >, std::allocator, std::allocator > > > const&, int)+0x1776) [0x563d52a75756] 8: (C_IO_Dir_OMAP_Fetched::finish(int)+0xbb) [0x563d52a87a1b] 9: (MDSContext::complete(int)+0x5f) [0x563d52b2a42f] 10: (MDSIOContextBase::complete(int)+0x524) [0x563d52b2abb4] 11: (Finisher::finisher_thread_entry()+0x18d) [0x7eff060d4f2d] 12: /lib64/libpthread.so.0(+0x81cf) [0x7eff04ddc1cf] 13: clone() 1 jobs: ['7184103'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/journal-repair}'] Failure: Test failure: test_ephemeral_pin_grow_mds (tasks.cephfs.test_exports.TestEphemeralPins) 1 jobs: ['7184090'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/exports}']