Found 249 jobs Missing teuthology log /home/teuthworker/archive/rishabh-2023-07-05_18:32:52-fs-wip-rishabh-CephManager-in-CephFSTestCase-testing-default-smithi/7327507/teuthology.log Misunderstood line: Misunderstood line: Found 23 distinct failure reasons Failure: Command failed (workunit test suites/dbench.sh) on smithi117 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0a781ef0807ae9473fb1306ef07036226ca4f1a6 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/dbench.sh' 1 jobs: ['7327527'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/dbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/5', 'objectstore-ec/bluestore-bitmap', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-no-extra-options}', 'wsync/no}'] Failure: reached maximum tries (51) after waiting for 300 seconds 2 jobs: ['7327358', '7327505'] suites intersection: ['1-ceph', '2-logrotate}'] suites union: ['1-ceph', '2-logrotate}', 'cephfs-mirror/three-per-cluster', 'clients/{mirror}', 'cluster/{1-node}', 'clusters/fixed-2-ucephfs', 'conf/{client', 'distro/{ubuntu_20.04}', 'fs/mirror-ha/{begin/{0-install', 'fs/traceless/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-comp', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{frag', 'overrides/{whitelist_health}', 'supported-random-distro$/{rhel_8}', 'tasks/cfuse_workunit_suites_dbench', 'traceless/50pc}', 'workloads/cephfs-mirror-ha-workunit}'] Failure: Test failure: test_generic (tasks.cephfs.tests_from_xfstests_dev.TestXFSTestsDev) 1 jobs: ['7327386'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/xfstests-dev}'] Failure: Test failure: test_snapshot_remove (tasks.cephfs.test_strays.TestStrays) 1 jobs: ['7327559'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/strays}'] Failure: error during scrub thrashing: rank damage found: {'backtrace'} 7 jobs: ['7327501', '7327472', '7327517', '7327418', '7327373', '7327467', '7327478'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'ranks/multi/{export-check', 'session_timeout}', 'standby-replay'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/fs/misc}}', '5-workunit/kernel_untar_build}}', '5-workunit/suites/blogbench}}', '5-workunit/suites/ffsb}}', '5-workunit/suites/iogen}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'rhel_8}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{no-subvolume}', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}', 'wsync/yes}'] Timeout 3h running clone.client.0/qa/workunits/suites/fsstress.sh 1 jobs: ['7327580'] suites: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/no', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/pacific'] Dead: None 6 jobs: ['7327507', '7327489', '7327518', '7327399', '7327366', '7327410'] Failure: Command failed on smithi114 with status 5: 'sudo systemctl stop ceph-46f4c4b8-1b87-11ee-9b2f-001a4aab830c@mon.smithi114' 6 jobs: ['7327538', '7327453', '7327409', '7327562', '7327415', '7327442'] suites intersection: ['conf/{client', 'mds', 'mon', 'osd}'] suites union: ['1-ceph', '1-cephadm', '1-check-counter', '1-client', '1-mount/mount/fuse', '1-ranks/2', '1-tests/fsstress}}}', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client', '2-logrotate}', '2-scrub/no', '2-upgrade', '2-workload/stress_tests/{0-client-upgrade', '3-compat_client/quincy}}', '3-inline/yes', '3-snaps/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-flush/no', '4-verify}', '5-upgrade-with-workload', '5-workunit/suites/iogen}}', '6-verify}}', 'begin/{0-install', 'centos_8.stream_container_tools', 'centos_latest', 'clusters/1-mds-2-client', 'clusters/1-mds-2-client-micro', 'clusters/1-mds-3-client', 'clusters/1a11s-mds-1c-client-3node', 'clusters/{1-mds-1-client-micro}', 'conf/{client', 'distros/ubuntu_latest', 'fail_fs/no', 'frag', 'fs/multiclient/{begin/{0-install', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'fs/upgrade/upgraded_client/{bluestore-bitmap', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'multimds/yes', 'n/3', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'pg-warn', 'pg-warn}', 'ranks/multi/{export-check', 'replication/default}', 'rhel_8}', 'roles', 'session_timeout}', 'standby-replay', 'syntax}', 'tasks/mdtest}', 'tasks/{0-from/nautilus', 'tasks/{0-from/pacific', 'tasks/{0-octopus', 'tasks/{0-subvolume/{with-quota}', 'wsync/yes}'] Failure: Command failed (workunit test suites/ffsb.sh) on smithi117 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0a781ef0807ae9473fb1306ef07036226ca4f1a6 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh' 2 jobs: ['7327577', '7327385'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/5', 'objectstore-ec/bluestore-bitmap', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/always}', 'session_timeout}', 'standby-replay', 'wsync/no}'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/5', 'objectstore-ec/bluestore-bitmap', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}'] Failure: Command failed on smithi203 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:0a781ef0807ae9473fb1306ef07036226ca4f1a6 pull' 3 jobs: ['7327440', '7327531', '7327524'] suites intersection: ['2-logrotate}', 'conf/{client', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-comp', 'osd}'] suites union: ['1-ceph', '1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/yes', '5-workunit/fs/norstats}}', '5-workunit/suites/fsx}}', 'begin/{0-install', 'clusters/1-mds-2-client', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'distros/ubuntu_latest', 'frag', 'fs/multiclient/{begin/{0-install', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'n/3', 'objectstore-ec/bluestore-comp', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{ignorelist_health', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/ior-shared-file}', 'tasks/{0-subvolume/{with-namespace-isolated}'] Valgrind: mon (InvalidRead, Leak_PossiblyLost, Leak_StillReachable) 2 jobs: ['7327582', '7327446'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{latest', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'osd}', 'overrides/{ignorelist_health', 'overrides}}', 'ranks/3', 'session_timeout}', 'tasks/fsstress', 'validater/valgrind}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{latest', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{ignorelist_health', 'overrides}}', 'ranks/3', 'session_timeout}', 'tasks/fsstress', 'validater/valgrind}'] Failure: "2023-07-05T22:51:38.959331+0000 mon.a (mon.0) 405 : cluster [WRN] Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY)" in cluster log 1 jobs: ['7327542'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/snap-schedule}'] Failure: {'smithi137.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'cmd': '/usr/bin/cpanm Amazon::S3', 'invocation': {'module_args': {'executable': None, 'from_path': None, 'installdeps': False, 'locallib': None, 'mirror': None, 'mirror_only': False, 'name': 'Amazon::S3', 'notest': False, 'system_lib': False, 'version': None}}, 'msg': "! Finding Amazon::S3 on cpanmetadb failed.\n! Finding LWP::UserAgent::Determined on cpanmetadb failed.\n! cannot open file '/root/.cpanm/sources/http%www.cpan.org/02packages.details.txt.gz': No such file or directory opening compressed index\n! Couldn't find module or a distribution LWP::UserAgent::Determined\n! Finding Digest::HMAC_SHA1 on cpanmetadb failed.\n! Installing the dependencies failed: Module 'LWP::UserAgent::Determined' is not installed\n! Bailing out the installation for Amazon-S3-0.63.\n"}, 'smithi097.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'cmd': '/usr/bin/cpanm Amazon::S3', 'invocation': {'module_args': {'executable': None, 'from_path': None, 'installdeps': False, 'locallib': None, 'mirror': None, 'mirror_only': False, 'name': 'Amazon::S3', 'notest': False, 'system_lib': False, 'version': None}}, 'msg': "! Finding Amazon::S3 on cpanmetadb failed.\n! Finding Digest::MD5::File on cpanmetadb failed.\n! cannot open file '/root/.cpanm/sources/http%www.cpan.org/02packages.details.txt.gz': No such file or directory opening compressed index\n! Couldn't find module or a distribution Digest::MD5::File (0.08)\n! Finding Test::More on cpanmetadb failed.\n! Installing the dependencies failed: Module 'Digest::MD5::File' is not installed\n! Bailing out the installation for Amazon-S3-0.63.\n"}} 1 jobs: ['7327492'] suites: ['1-ceph', '2-logrotate}', '2-workunit/iozone}}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/thrash/multifs/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'msgr-failures/osd-mds-delay', 'multifs', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{frag', 'session_timeout', 'tasks/{1-thrash/mon', 'thrashosds-health}'] Dead: b'2023-07-06T11:17:41.035 DEBUG:teuthology.exit:Finished running handlers' ceph version 18.0.0-4488-g0a781ef0 (0a781ef0807ae9473fb1306ef07036226ca4f1a6) reef (dev) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f897f842520] 2: pthread_kill() 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7f898016173b] 6: /usr/lib/ceph/libceph-common.so.2(+0x161895) [0x7f8980161895] 7: ceph-mds(+0x21b0bb) [0x562aec0620bb] 8: (Locker::xlock_finish(std::_Rb_tree_const_iterator const&, MutationImpl*, bool*)+0x8d) [0x562aec11b6fd] 9: (Locker::_drop_locks(MutationImpl*, std::set, std::allocator >*, bool)+0x47d) [0x562aec10da5d] 10: (Locker::drop_non_rdlocks(MutationImpl*, std::set, std::allocator >*)+0x63) [0x562aec10e153] 11: (Server::reply_client_request(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x428) [0x562aebfefad8] 12: (Server::respond_to_request(boost::intrusive_ptr&, int)+0x11e) [0x562aebfe4cbe] 13: (Server::_unlink_local_finish(boost::intrusive_ptr&, CDentry*, CDentry*, unsigned long)+0x310) [0x562aec01c5d0] 14: (MDSContext::complete(int)+0x5c) [0x562aec20a51c] 15: (MDSIOContextBase::complete(int)+0x30c) [0x562aec20a94c] 16: (MDSLogContextBase::complete(int)+0x53) [0x562aec20aa83] 17: (Finisher::finisher_thread_entry()+0x175) [0x7f8980214755] 18: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7f897f894b43] 19: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7f897f926a00] 1 jobs: ['7327581'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/xfstests-dev}'] Dead: b'2023-07-06T07:26:31.593 DEBUG:teuthology.exit:Finished running handlers' 4 jobs: ['7327413', '7327405', '7327486', '7327563'] suites intersection: ['2-logrotate}', 'conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}'] suites union: ['1-ceph', '1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/yes', '4-flush/no', '5-workunit/direct_io}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'frag', 'fs/multifs/{begin/{0-install', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v2}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/5', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'ranks/multi/{export-check', 'replication/default}', 'rhel_8}', 'session_timeout}', 'standby-replay', 'tasks/multifs-auth}', 'tasks/{0-subvolume/{with-quota}', 'wsync/yes}'] Failure: Command failed on smithi027 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=18.0.0-4488-g0a781ef0-1jammy cephadm=18.0.0-4488-g0a781ef0-1jammy ceph-mds=18.0.0-4488-g0a781ef0-1jammy ceph-mgr=18.0.0-4488-g0a781ef0-1jammy ceph-common=18.0.0-4488-g0a781ef0-1jammy ceph-fuse=18.0.0-4488-g0a781ef0-1jammy ceph-test=18.0.0-4488-g0a781ef0-1jammy ceph-volume=18.0.0-4488-g0a781ef0-1jammy radosgw=18.0.0-4488-g0a781ef0-1jammy python3-rados=18.0.0-4488-g0a781ef0-1jammy python3-rgw=18.0.0-4488-g0a781ef0-1jammy python3-cephfs=18.0.0-4488-g0a781ef0-1jammy python3-rbd=18.0.0-4488-g0a781ef0-1jammy libcephfs2=18.0.0-4488-g0a781ef0-1jammy libcephfs-dev=18.0.0-4488-g0a781ef0-1jammy librados2=18.0.0-4488-g0a781ef0-1jammy librbd1=18.0.0-4488-g0a781ef0-1jammy rbd-fuse=18.0.0-4488-g0a781ef0-1jammy python3-cephfs=18.0.0-4488-g0a781ef0-1jammy cephfs-shell=18.0.0-4488-g0a781ef0-1jammy cephfs-top=18.0.0-4488-g0a781ef0-1jammy cephfs-mirror=18.0.0-4488-g0a781ef0-1jammy' 1 jobs: ['7327485'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-3-client', 'conf/{client', 'distros/ubuntu_latest', 'fs/multiclient/{begin/{0-install', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/cephfs_misc_tests}'] Failure: Command failed (workunit test fs/quota/quota.sh) on smithi162 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0a781ef0807ae9473fb1306ef07036226ca4f1a6 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 CEPH_MNT=/home/ubuntu/cephtest/mnt.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/fs/quota/quota.sh' 2 jobs: ['7327586', '7327389'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/workunit/quota}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{ubuntu_20.04}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/workunit/quota}'] Failure: Test failure: test_disable_enable_human_readable_quota_values (tasks.cephfs.test_quota.TestQuota) 1 jobs: ['7327521'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/quota}'] Failure: Command failed on smithi186 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 96321d18-1b7e-11ee-9b2f-001a4aab830c -- ceph mon dump -f json' 1 jobs: ['7327424'] suites: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/yes', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/pacific'] Failure: Command failed (workunit test kernel_untar_build.sh) on smithi135 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0a781ef0807ae9473fb1306ef07036226ca4f1a6 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 CEPH_MNT=/home/ubuntu/cephtest/mnt.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/kernel_untar_build.sh' 2 jobs: ['7327551', '7327460'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu/{latest', 'fs/mixed-clients/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'kclient-overrides/{distro/testing/k-testing', 'mds', 'mon', 'ms-die-on-skipped}', 'osd-asserts}', 'osd}', 'overrides/{ignorelist_health', 'overrides}}', 'tasks/kernel_cfuse_workunits_untarbuild_blogbench}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu/{latest', 'fs/mixed-clients/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'kclient-overrides/{distro/testing/k-testing', 'mds', 'mon', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'osd-asserts}', 'osd}', 'overrides/{ignorelist_health', 'overrides}}', 'tasks/kernel_cfuse_workunits_untarbuild_blogbench}'] Failure: Command failed (workunit test fs/test_python.sh) on smithi079 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0a781ef0807ae9473fb1306ef07036226ca4f1a6 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/test_python.sh' 1 jobs: ['7327546'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-1-client-coloc', 'conf/{client', 'distro/{rhel_8}', 'fs/libcephfs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/libcephfs_python}'] Failure: Command failed on smithi007 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install bison flex libelf-dev libssl-dev network-manager iproute2 util-linux dump indent libaio-dev libtool-bin uuid-dev xfslibs-dev postgresql postgresql-client postgresql-common postgresql-contrib' 1 jobs: ['7327488'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-1c-client', 'conf/{client', 'distro/{ubuntu_20.04}', 'fs/snaps/{begin/{0-install', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/workunit/snaps}'] Failure: Command failed on smithi195 with status 2: 'TESTDIR=/home/ubuntu/cephtest bash -s' 1 jobs: ['7327404'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-3-client', 'conf/{client', 'distros/ubuntu_latest', 'fs/multiclient/{begin/{0-install', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/ior-shared-file}']