Found 55 jobs Found 6 distinct failure reasons Failure: Command failed on smithi067 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=14.2.15-254-g4465473f-1xenial ceph-mds=14.2.15-254-g4465473f-1xenial ceph-mgr=14.2.15-254-g4465473f-1xenial ceph-common=14.2.15-254-g4465473f-1xenial ceph-fuse=14.2.15-254-g4465473f-1xenial ceph-test=14.2.15-254-g4465473f-1xenial radosgw=14.2.15-254-g4465473f-1xenial python-ceph=14.2.15-254-g4465473f-1xenial libcephfs2=14.2.15-254-g4465473f-1xenial libcephfs-dev=14.2.15-254-g4465473f-1xenial librados2=14.2.15-254-g4465473f-1xenial librbd1=14.2.15-254-g4465473f-1xenial rbd-fuse=14.2.15-254-g4465473f-1xenial' 15 jobs: ['5668777', '5668758', '5668791', '5668799', '5668783', '5668759', '5668772', '5668760', '5668792', '5668770', '5668781', '5668793', '5668785', '5668762', '5668753'] suites intersection: ['clusters/{fixed-2', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'supported-random-distro$/{ubuntu_16.04}', 'thrashosds-health'] suites union: ['clusters/{fixed-2', 'fast/fast', 'fast/normal', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-delay', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'recovery-overrides/{default}', 'recovery-overrides/{more-active-recovery}', 'recovery-overrides/{more-async-partial-recovery}', 'recovery-overrides/{more-async-recovery}', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{ubuntu_16.04}', 'thrashers/careful', 'thrashers/default', 'thrashers/fastread', 'thrashers/morepggrow', 'thrashers/pggrow', 'thrashosds-health', 'workloads/ec-radosbench}', 'workloads/ec-small-objects-fast-read}', 'workloads/ec-small-objects-many-deletes}', 'workloads/ec-small-objects}'] Crash: timed out waiting for admin_socket to appear after osd.3 restart ceph version 14.2.15-254-g4465473 (4465473f4beb0abf5eea83640844c5ce6c4a4d1d) nautilus (stable) 1: (()+0x12980) [0x7f326ddf4980] 2: (gsignal()+0xc7) [0x7f326ced2fb7] 3: (abort()+0x141) [0x7f326ced4921] 4: (()+0x3048a) [0x7f326cec448a] 5: (()+0x30502) [0x7f326cec4502] 6: (crimson::dmclock::PriorityQueueBase::do_add_request(std::unique_ptr >&&, ceph::mclock::osd_op_type_t const&, crimson::dmclock::ReqParams const&, double, unsigned int, crimson::dmclock::ClientInfo const&)+0x105c) [0x55fdb2a03bdc] 7: (crimson::dmclock::PullPriorityQueue::add_request(OpQueueItem&&, ceph::mclock::osd_op_type_t const&, unsigned int)+0x13f) [0x55fdb2a03e1f] 8: (ceph::mClockOpClassQueue::enqueue(unsigned long, unsigned int, unsigned int, OpQueueItem&&)+0x45) [0x55fdb2a03f55] 9: (OSD::ShardedOpWQ::_enqueue(OpQueueItem&&)+0xc3) [0x55fdb274efd3] 10: (OSD::enqueue_op(spg_t, boost::intrusive_ptr&&, unsigned int)+0x1ca) [0x55fdb274f41a] 11: (OSD::ms_fast_dispatch(Message*)+0x191) [0x55fdb2754661] 12: (DispatchQueue::fast_dispatch(boost::intrusive_ptr const&)+0x27d) [0x55fdb314850d] 13: (ProtocolV2::handle_message()+0xafa) [0x55fdb31a8b0a] 14: (ProtocolV2::handle_read_frame_dispatch()+0x258) [0x55fdb31be0f8] 15: (ProtocolV2::_handle_read_frame_epilogue_main()+0x69) [0x55fdb31be2c9] 16: (ProtocolV2::handle_read_frame_epilogue_main(std::unique_ptr&&, int)+0x99) [0x55fdb31bffa9] 17: (ProtocolV2::run_continuation(Ct&)+0x34) [0x55fdb31a3e54] 18: (AsyncConnection::process()+0x186) [0x55fdb3177f06] 19: (EventCenter::process_events(unsigned int, std::chrono::duration >*)+0x7ed) [0x55fdb2faeefd] 20: (()+0xdc5468) [0x55fdb2fb1468] 21: (()+0xbd6df) [0x7f326d8f86df] 22: (()+0x76db) [0x7f326dde96db] 23: (clone()+0x3f) [0x7f326cfb571f] 6 jobs: ['5668774', '5668767', '5668796', '5668751', '5668763', '5668787'] suites intersection: ['clusters/{fixed-2', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'supported-random-distro$/{ubuntu_latest}', 'thrashosds-health'] suites union: ['clusters/{fixed-2', 'fast/fast', 'fast/normal', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-delay', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'recovery-overrides/{default}', 'recovery-overrides/{more-active-recovery}', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/careful', 'thrashers/default', 'thrashers/fastread', 'thrashers/pggrow', 'thrashosds-health', 'workloads/ec-small-objects-fast-read}', 'workloads/ec-small-objects-many-deletes}', 'workloads/ec-small-objects}'] Dead: b'2020-11-30T13:37:45.689 DEBUG:teuthology.exit:Finished running handlers' 2 jobs: ['5668794', '5668798'] suites intersection: ['clusters/{fixed-2', 'fast/normal', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'supported-random-distro$/{ubuntu_latest}', 'thrashosds-health', 'workloads/ec-small-objects-fast-read}'] suites union: ['clusters/{fixed-2', 'fast/normal', 'msgr-failures/fastclose', 'msgr-failures/few', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'recovery-overrides/{default}', 'recovery-overrides/{more-active-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/default', 'thrashers/fastread', 'thrashosds-health', 'workloads/ec-small-objects-fast-read}'] Failure: SSH connection to smithi186 was lost: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --no-omap --ec-pool --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 0 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --op setattr 25 --op rmattr 25 --op append 50 --op write_excl 0 --op append_excl 50 --pool unique_pool_0' 1 jobs: ['5668780'] suites: ['clusters/{fixed-2', 'fast/normal', 'msgr-failures/few', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{rhel_latest}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/ec-small-objects}'] Failure: reached maximum tries (800) after waiting for 4800 seconds 3 jobs: ['5668797', '5668769', '5668765'] suites intersection: ['clusters/{fixed-2', 'fast/fast', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'supported-random-distro$/{ubuntu_latest}', 'thrashosds-health', 'workloads/ec-radosbench}'] suites union: ['clusters/{fixed-2', 'fast/fast', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-delay', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'recovery-overrides/{more-active-recovery}', 'recovery-overrides/{more-async-partial-recovery}', 'recovery-overrides/{more-async-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/careful', 'thrashers/fastread', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/ec-radosbench}'] Dead: b'2020-11-30T13:37:05.772 INFO:teuthology.orchestra.run.smithi144.stdout: "event": "throttled"' 1 jobs: ['5668764'] suites: ['clusters/{fixed-2', 'fast/normal', 'msgr-failures/fastclose', 'objectstore/bluestore-comp-lz4', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'recovery-overrides/{more-async-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/fastread', 'thrashosds-health', 'workloads/ec-small-objects}']