Found 172 jobs Found 8 distinct failure reasons Timeout 3h running clone.client.0/qa/workunits/rbd/qemu_dynamic_features.sh 2 jobs: ['5918207', '5918338'] suites intersection: ['clusters/{fixed-3', 'openstack}', 'qemu/xfstests', 'rbd/maintenance/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'workloads/dynamic_features_no_cache}'] suites union: ['clusters/{fixed-3', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'openstack}', 'qemu/xfstests', 'rbd/maintenance/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'workloads/dynamic_features_no_cache}'] Crash: "2021-02-27T08:45:23.387590+0000 mon.a (mon.0) 516 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log ceph version 16.1.0-471-ge9e1dc3c (e9e1dc3c02bc2ae55fd7c1788cad0c2883ee19e5) pacific (rc) 1: /lib64/libpthread.so.0(+0x12b20) [0x7f5dfe5a5b20] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9009b) [0x7f5dfdbc309b] 5: /lib64/libstdc++.so.6(+0x9653c) [0x7f5dfdbc953c] 6: /lib64/libstdc++.so.6(+0x96597) [0x7f5dfdbc9597] 7: /lib64/libstdc++.so.6(+0x967f8) [0x7f5dfdbc97f8] 8: /lib64/libtcmalloc.so.4(+0x19d24) [0x7f5dff312d24] 9: (tcmalloc::allocate_full_cpp_throw_oom(unsigned long)+0x146) [0x7f5dff3350d6] 10: (rocksdb::Arena::AllocateNewBlock(unsigned long)+0x43) [0x556816d7caf3] 11: (rocksdb::Arena::AllocateFallback(unsigned long, bool)+0x4b) [0x556816d7cb9b] 12: (rocksdb::ConcurrentArena::AllocateAligned(unsigned long, unsigned long, rocksdb::Logger*)+0xb4) [0x556816cfe634] 13: ceph-osd(+0x12276ee) [0x556816d7d6ee] 14: (rocksdb::MemTable::Add(unsigned long, rocksdb::ValueType, rocksdb::Slice const&, rocksdb::Slice const&, bool, rocksdb::MemTablePostProcessInfo*, void**)+0xd1) [0x556816cf9451] 15: (rocksdb::MemTableInserter::PutCFImpl(unsigned int, rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::ValueType)+0x2e4) [0x556816d605e4] 16: (rocksdb::MemTableInserter::PutCF(unsigned int, rocksdb::Slice const&, rocksdb::Slice const&)+0x17) [0x556816d60d87] 17: (rocksdb::WriteBatchInternal::Iterate(rocksdb::WriteBatch const*, rocksdb::WriteBatch::Handler*, unsigned long, unsigned long)+0x5ea) [0x556816d56d3a] 18: (rocksdb::WriteBatch::Iterate(rocksdb::WriteBatch::Handler*) const+0x6a) [0x556816d5726a] 19: (rocksdb::WriteBatchInternal::InsertInto(rocksdb::WriteThread::WriteGroup&, unsigned long, rocksdb::ColumnFamilyMemTables*, rocksdb::FlushScheduler*, rocksdb::TrimHistoryScheduler*, bool, unsigned long, rocksdb::DB*, bool, bool, bool)+0x161) [0x556816d5aec1] 20: (rocksdb::DBImpl::WriteImpl(rocksdb::WriteOptions const&, rocksdb::WriteBatch*, rocksdb::WriteCallback*, unsigned long*, unsigned long, bool, unsigned long*, unsigned long, rocksdb::PreReleaseCallback*)+0x1c11) [0x556816c806c1] 21: (rocksdb::DBImpl::Write(rocksdb::WriteOptions const&, rocksdb::WriteBatch*)+0x21) [0x556816c812d1] 22: (RocksDBStore::submit_common(rocksdb::WriteOptions&, std::shared_ptr)+0x84) [0x556816c20b84] 23: (RocksDBStore::submit_transaction(std::shared_ptr)+0x8e) [0x556816c2140e] 24: (BlueStore::_txc_apply_kv(BlueStore::TransContext*, bool)+0x436) [0x5568166e6706] 25: (BlueStore::_kv_sync_thread()+0x16dc) [0x55681671f07c] 26: (BlueStore::KVSyncThread::entry()+0x11) [0x556816747391] 27: /lib64/libpthread.so.0(+0x814a) [0x7f5dfe59b14a] 28: clone() 1 jobs: ['5918220'] suites: ['2-clusters/{fixed-3', '3-objectstore/bluestore-hybrid', '4-supported-random-distro$/{rhel_8}', '5-pool/replicated-data-pool', '6-prepare/qcow2-http', '7-io-workloads/qemu_xfstests', '8-migrate-workloads/execute', '9-cleanup/cleanup}', 'openstack}', 'rbd/migration/{1-base/install'] Failure: SELinux denials found on ubuntu@smithi101.front.sepia.ceph.com: ['type=AVC msg=audit(1614414145.886:5614): avc: denied { read } for pid=891 comm="sssd" name="resolv.conf" dev="sda1" ino=265261 scontext=system_u:system_r:sssd_t:s0 tcontext=unconfined_u:object_r:admin_home_t:s0 tclass=file permissive=1'] 26 jobs: ['5918283', '5918361', '5918202', '5918364', '5918342', '5918281', '5918327', '5918359', '5918259', '5918358', '5918368', '5918257', '5918248', '5918270', '5918242', '5918320', '5918345', '5918304', '5918230', '5918246', '5918267', '5918239', '5918351', '5918294', '5918367', '5918208'] suites intersection: [] suites union: ['2-clusters/{fixed-3', '3-objectstore/bluestore-comp-lz4', '3-objectstore/bluestore-comp-zstd', '4-supported-random-distro$/{rhel_8}', '5-pool/none', '6-prepare/qcow2-file', '6-prepare/raw-file', '7-io-workloads/qemu_xfstests', '8-migrate-workloads/execute', '9-cleanup/cleanup}', 'clients/mirror', 'clients/{mirror-extra', 'cluster/{2-node', 'cluster/{fixed-3', 'clusters/{fixed-1', 'clusters/{fixed-2', 'clusters/{fixed-3', 'config/none', 'features/defaults', 'features/format-1', 'features/readbalance', 'min-compat-client/default', 'min-compat-client/octopus', 'mirror}', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'objectstore/filestore-xfs', 'openstack', 'openstack}', 'policy/simple', 'pool/ec-cache-pool', 'pool/ec-data-pool', 'pool/none', 'pool/small-cache-pool', 'qemu/xfstests', 'rbd-mirror/four-per-cluster', 'rbd/cli_v1/{base/install', 'rbd/encryption/{cache/writearound', 'rbd/librbd/{cache/writearound', 'rbd/librbd/{cache/writeback', 'rbd/maintenance/{base/install', 'rbd/migration/{1-base/install', 'rbd/mirror-thrash/{base/install', 'rbd/mirror/{base/install', 'rbd/nbd/{base/install', 'rbd/qemu/{cache/none', 'rbd/qemu/{cache/writearound', 'rbd/qemu/{cache/writethrough', 'rbd/singleton-bluestore/{all/issue-20295', 'rbd/singleton/{all/formatted-output', 'rbd/singleton/{all/qemu-iotests-no-cache', 'rbd/singleton/{all/rbd-vs-unmanaged-snaps', 'rbd/thrash/{base/install', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{rhel_8}}', 'thrashers/cache', 'thrashers/default', 'thrashosds-health', 'workloads/c_api_tests}', 'workloads/dynamic_features_no_cache}', 'workloads/dynamic_features}', 'workloads/fsx}', 'workloads/qemu_fsstress}', 'workloads/qemu_xfstests_luks1}', 'workloads/qemu_xfstests}', 'workloads/rbd-mirror-fsx-workunit}', 'workloads/rbd-mirror-snapshot-workunit-minimum}', 'workloads/rbd-mirror-workunit-min-compat-client-octopus}', 'workloads/rbd_cli_import_export}', 'workloads/rbd_fsx_cache_writeback}', 'workloads/rbd_fsx_nbd}', 'workloads/rbd_nbd}', 'workloads/rebuild_object_map}'] Failure: unexpected end of data 1 jobs: ['5918274'] suites: ['clusters/{fixed-3', 'objectstore/bluestore-hybrid', 'openstack}', 'qemu/xfstests', 'rbd/maintenance/{base/install', 'supported-random-distro$/{rhel_8}', 'workloads/dynamic_features}'] Failure: Command failed on smithi049 with status 1: 'qemu-img convert -f qcow2 -O raw /home/ubuntu/cephtest/qemu/base.client.0.0.qcow2 rbd:rbd/client.0.0' 11 jobs: ['5918269', '5918318', '5918356', '5918328', '5918339', '5918286', '5918223', '5918319', '5918303', '5918287', '5918222'] suites intersection: ['openstack}'] suites union: ['2-clusters/{fixed-3', '3-objectstore/bluestore-bitmap', '3-objectstore/bluestore-comp-snappy', '3-objectstore/bluestore-comp-zlib', '3-objectstore/bluestore-comp-zstd', '4-supported-random-distro$/{centos_8}', '5-pool/ec-data-pool', '5-pool/none', '5-pool/replicated-data-pool', '6-prepare/qcow2-file', '6-prepare/qcow2-http', '6-prepare/raw-file', '7-io-workloads/qemu_xfstests', '8-migrate-workloads/execute', '9-cleanup/cleanup}', 'clusters/{fixed-3', 'features/defaults', 'features/journaling', 'features/readbalance', 'msgr-failures/few', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/filestore-xfs', 'openstack}', 'pool/ceph_and_immutable_object_cache', 'pool/ec-cache-pool', 'pool/small-cache-pool', 'qemu/xfstests', 'rbd/encryption/{cache/none', 'rbd/encryption/{cache/writethrough', 'rbd/immutable-object-cache/{clusters/{fix-2', 'rbd/maintenance/{base/install', 'rbd/migration/{1-base/install', 'rbd/qemu/{cache/writearound', 'rbd/qemu/{cache/writethrough', 'supported-random-distro$/{centos_8}', 'workloads/dynamic_features_no_cache}', 'workloads/qemu_bonnie}', 'workloads/qemu_on_immutable_object_cache_and_thrash}', 'workloads/qemu_xfstests_luks1}', 'workloads/qemu_xfstests_luks2}', 'workloads/rebuild_object_map}'] Crash: "2021-02-27T07:23:21.017211+0000 mon.a (mon.0) 1038 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log ceph version 16.1.0-471-ge9e1dc3c (e9e1dc3c02bc2ae55fd7c1788cad0c2883ee19e5) pacific (rc) 1: /lib64/libpthread.so.0(+0x12b20) [0x7fddeba2cb20] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9009b) [0x7fddeb04a09b] 5: /lib64/libstdc++.so.6(+0x9653c) [0x7fddeb05053c] 6: /lib64/libstdc++.so.6(+0x96597) [0x7fddeb050597] 7: /lib64/libstdc++.so.6(+0x967f8) [0x7fddeb0507f8] 8: ceph-osd(+0x587fa6) [0x55d33647ffa6] 9: (ceph::buffer::v15_2_0::create_aligned(unsigned int, unsigned int)+0x26) [0x55d336d93e46] 10: (ceph::buffer::v15_2_0::create(unsigned int)+0x26) [0x55d336d93e96] 11: (ceph::buffer::v15_2_0::list::iterator_impl::copy_shallow(unsigned int, ceph::buffer::v15_2_0::ptr&)+0x5a) [0x55d336d98c0a] 12: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x1abb) [0x55d336aa709b] 13: (BlueStore::queue_transactions(boost::intrusive_ptr&, std::vector >&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x316) [0x55d336aa8996] 14: (non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector >&, boost::intrusive_ptr)+0x58) [0x55d336704958] 15: (ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr >&&, eversion_t const&, eversion_t const&, std::vector >&&, std::optional&, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr)+0xc9d) [0x55d3368eebcd] 16: (PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0xce0) [0x55d3366706a0] 17: (PrimaryLogPG::execute_ctx(PrimaryLogPG::OpContext*)+0x115d) [0x55d3366cbf3d] 18: (PrimaryLogPG::do_op(boost::intrusive_ptr&)+0x2d50) [0x55d3366d4a20] 19: (PrimaryLogPG::do_request(boost::intrusive_ptr&, ThreadPool::TPHandle&)+0xd1c) [0x55d3366dbd8c] 20: (OSD::dequeue_op(boost::intrusive_ptr, boost::intrusive_ptr, ThreadPool::TPHandle&)+0x309) [0x55d336565d79] 21: (ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr&, ThreadPool::TPHandle&)+0x68) [0x55d3367c2088] 22: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xa58) [0x55d3365851c8] 23: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x55d336bea5e4] 24: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x55d336bed284] 25: /lib64/libpthread.so.0(+0x814a) [0x7fddeba2214a] 26: clone() 1 jobs: ['5918201'] suites: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-comp-zlib', 'openstack}', 'policy/none', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{rhel_8}', 'workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock}'] Failure: Command failed (workunit test rbd/qemu-iotests.sh) on smithi191 with status 13: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=e9e1dc3c02bc2ae55fd7c1788cad0c2883ee19e5 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/qemu-iotests.sh' 2 jobs: ['5918349', '5918333'] suites intersection: ['openstack', 'supported-random-distro$/{centos_8}}'] suites union: ['objectstore/bluestore-comp-zlib', 'objectstore/bluestore-hybrid', 'openstack', 'rbd/singleton/{all/qemu-iotests-writearound', 'rbd/singleton/{all/qemu-iotests-writethrough', 'supported-random-distro$/{centos_8}}'] Failure: Command failed on smithi170 with status 123: "sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --" 2 jobs: ['5918234', '5918363'] suites intersection: ['openstack}'] suites union: ['centos_latest', 'clusters/{fixed-1', 'clusters/{fixed-3', 'features/defaults', 'msgr-failures/few', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-stupid', 'openstack}', 'pool/ec-cache-pool', 'rbd/encryption/{cache/writethrough', 'rbd/valgrind/{base/install', 'supported-random-distro$/{rhel_8}', 'validator/memcheck', 'workloads/c_api_tests}', 'workloads/qemu_xfstests_luks1}']