Found 343 jobs Found 8 distinct failure reasons Failure: Command failed on smithi199 with status 127: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:87cd59fd3ebb82c3ebadd5335e99940fbb5394cc shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 88dd262a-81df-11eb-9080-001a4aab830c -- ceph orch host ls --format=json' 5 jobs: ['5952020', '5951855', '5952023', '5952028', '5952013'] suites intersection: [] suites union: ['1-start', '2-thrash', '3-tasks/rados_api_tests', 'fixed-2', 'mode/root', 'mon_election/connectivity', 'msgr/async', 'msgr/async-v1only', 'rados/cephadm/smoke-roleless/{distro/ubuntu_20.04_kubic_testing', 'rados/cephadm/smoke/{distro/ubuntu_20.04_kubic_stable', 'rados/cephadm/thrash/{0-distro/ubuntu_20.04_kubic_stable', 'rados/cephadm/thrash/{0-distro/ubuntu_20.04_kubic_testing', 'rados/cephadm/with-work/{0-distro/ubuntu_20.04_kubic_stable', 'root}', 'start', 'start}', 'tasks/rados_python}'] Dead: b'2021-03-11T07:31:38.485 DEBUG:teuthology.exit:Finished running handlers' 3 jobs: ['5951915', '5951951', '5951891'] suites intersection: ['msgr-failures/few', 'rados'] suites union: ['ceph', 'clusters/{fixed-2', 'd-thrash/default/{default', 'fast/fast', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async', 'msgr/async-v2only', 'objectstore/bluestore-comp-zlib', 'objectstore/filestore-xfs', 'openstack}', 'rados', 'rados/singleton/{all/thrash-eio', 'rados/thrash-erasure-code-overwrites/{bluestore-bitmap', 'rados/verify/{centos_latest', 'recovery-overrides/{more-async-partial-recovery}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}}', 'tasks/rados_api_tests', 'thrashers/careful', 'thrashosds-health', 'thrashosds-health}', 'validater/valgrind}', 'workloads/ec-small-objects-overwrites}'] Failure: Test failure: test_create_and_delete_export (tasks.cephfs.test_nfs.TestNFS) 1 jobs: ['5951645'] suites: ['mon_election/classic', 'rados/cephadm/workunits/{0-distro/ubuntu_20.04_kubic_testing', 'task/test_orch_cli}'] Crash: Command failed (workunit test rados/test_rados_tool.sh) on smithi076 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=af18f5db7a2f7fd441af163239df24e765d7f3ad TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test_rados_tool.sh' ceph version 16.1.0-630-g87cd59fd (87cd59fd3ebb82c3ebadd5335e99940fbb5394cc) pacific (rc) 1: /lib64/libpthread.so.0(+0x12dc0) [0x7fef7776adc0] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9006b) [0x7fef76d8d06b] 5: /lib64/libstdc++.so.6(+0x9650c) [0x7fef76d9350c] 6: /lib64/libstdc++.so.6(+0x96567) [0x7fef76d93567] 7: /lib64/libstdc++.so.6(+0x967c8) [0x7fef76d937c8] 8: /lib64/libtcmalloc.so.4(+0x19d24) [0x7fef784d4d24] 9: (tcmalloc::allocate_full_cpp_throw_oom(unsigned long)+0x146) [0x7fef784f70d6] 10: (std::__cxx11::basic_string, std::allocator >::_M_mutate(unsigned long, unsigned long, char const*, unsigned long)+0x6d) [0x558e0936488d] 11: (std::__cxx11::basic_string, std::allocator >::_M_append(char const*, unsigned long)+0x6b) [0x558e09364ccb] 12: (rocksdb::WriteBatchInternal::Put(rocksdb::WriteBatch*, unsigned int, rocksdb::Slice const&, rocksdb::Slice const&)+0x215) [0x558e09f478f5] 13: (rocksdb::WriteBatch::Put(rocksdb::ColumnFamilyHandle*, rocksdb::Slice const&, rocksdb::Slice const&)+0x32) [0x558e09f47b32] 14: (RocksDBStore::RocksDBTransactionImpl::put_bat(rocksdb::WriteBatch&, rocksdb::ColumnFamilyHandle*, std::__cxx11::basic_string, std::allocator > const&, ceph::buffer::v15_2_0::list const&)+0x18b) [0x558e09dfe4fb] 15: (RocksDBStore::RocksDBTransactionImpl::set(std::__cxx11::basic_string, std::allocator > const&, std::__cxx11::basic_string, std::allocator > const&, ceph::buffer::v15_2_0::list const&)+0x54) [0x558e09e01fc4] 16: (BlueStore::_omap_setkeys(BlueStore::TransContext*, boost::intrusive_ptr&, boost::intrusive_ptr&, ceph::buffer::v15_2_0::list&)+0x54c) [0x558e098bab6c] 17: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x2336) [0x558e098f2ba6] 18: (BlueStore::queue_transactions(boost::intrusive_ptr&, std::vector >&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x316) [0x558e098f3c26] 19: (non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector >&, boost::intrusive_ptr)+0x58) [0x558e0954fbe8] 20: (ReplicatedBackend::do_repop(boost::intrusive_ptr)+0xeb0) [0x558e0973d880] 21: (ReplicatedBackend::_handle_message(boost::intrusive_ptr)+0x267) [0x558e0974dbd7] 22: (PGBackend::handle_message(boost::intrusive_ptr)+0x52) [0x558e095835f2] 23: (PrimaryLogPG::do_request(boost::intrusive_ptr&, ThreadPool::TPHandle&)+0x5de) [0x558e095268de] 24: (OSD::dequeue_op(boost::intrusive_ptr, boost::intrusive_ptr, ThreadPool::TPHandle&)+0x309) [0x558e093b1279] 25: (ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr&, ThreadPool::TPHandle&)+0x68) [0x558e0960d318] 26: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xa58) [0x558e093d06c8] 27: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x558e09a36fd4] 28: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x558e09a39c74] 29: /lib64/libpthread.so.0(+0x82de) [0x7fef777602de] 30: clone() 1 jobs: ['5951761'] suites: ['mon_election/connectivity', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-comp-zstd', 'rados', 'rados/singleton/{all/radostool', 'supported-random-distro$/{centos_8}}'] Failure: Command failed (workunit test mon/pg_autoscaler.sh) on smithi070 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=af18f5db7a2f7fd441af163239df24e765d7f3ad TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/mon/pg_autoscaler.sh' 2 jobs: ['5951723', '5951705'] suites intersection: ['rados'] suites union: ['mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/many', 'msgr/async', 'msgr/async-v1only', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'rados', 'rados/singleton/{all/pg-autoscaler', 'rados/singleton/{all/pg-autoscaler-progress-off', 'supported-random-distro$/{rhel_8}}', 'supported-random-distro$/{ubuntu_latest}}'] Failure: Command failed (workunit test cephadm/create_iscsi_disks.sh) on smithi106 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=af18f5db7a2f7fd441af163239df24e765d7f3ad TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/cephadm/create_iscsi_disks.sh' 4 jobs: ['5951865', '5951530', '5951697', '5951983'] suites intersection: ['task/test_e2e}'] suites union: ['rados/cephadm/dashboard/{0-distro/ubuntu_20.04_kubic_stable', 'rados/cephadm/dashboard/{0-distro/ubuntu_20.04_kubic_testing', 'task/test_e2e}'] Failure: Scrubbing terminated -- not all pgs were active and clean. 1 jobs: ['5952022'] suites: ['mon_election/classic', 'msgr-failures/many', 'msgr/async-v1only', 'objectstore/bluestore-hybrid', 'rados', 'rados/singleton/{all/lost-unfound-delete', 'supported-random-distro$/{rhel_8}}'] Failure: "2021-03-10T20:34:40.013582+0000 mon.a (mon.0) 10 : cluster [WRN] Health check update: 1/3 mons down, quorum a,b (MON_DOWN)" in cluster log 1 jobs: ['5952024'] suites: ['clusters/{fixed-4', 'mon_election/connectivity', 'msgr-failures/few', 'objectstore/bluestore-comp-lz4', 'openstack}', 'rados', 'rados/thrash-erasure-code-shec/{ceph', 'recovery-overrides/{more-async-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/default', 'thrashosds-health', 'workloads/ec-rados-plugin=shec-k=4-m=3-c=2}']