2024-04-15T22:37:41.491 INFO:root:teuthology version: 0.0.1.dev249+gcd45576
2024-04-15T22:37:41.492 INFO:teuthology.results:seed: '4083'
2024-04-15T22:39:30.741 INFO:root:teuthology version: 0.0.1.dev249+gcd45576
2024-04-15T22:39:30.745 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish...
2024-04-16T02:09:47.434 INFO:teuthology.results:Tests finished! gathering results...
2024-04-16T02:09:47.573 INFO:scrape:Found 2 jobs
2024-04-16T02:09:49.152 INFO:scrape:Found 2 distinct failure reasons
2024-04-16T02:09:49.152 INFO:scrape:Crash: rgw multisite test failures
2024-04-16T02:09:49.152 INFO:scrape:ceph version 19.0.0-3084-g87dbfb63 (87dbfb632e974cb2c28d17423071d2d833b53b29) squid (dev)
 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f502b87a520]
 2: /lib/librados.so.2(+0x9ce65) [0x7f502d36ce65]
 3: (librados::v14_2_0::IoCtx::operate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::ObjectWriteOperation*, int, opentelemetry::v1::trace::SpanContext const*)+0x83) [0x7f502d34e903]
 4: (rgw_rados_operate(DoutPrefixProvider const*, librados::v14_2_0::IoCtx&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::ObjectWriteOperation*, optional_yield, int, opentelemetry::v1::trace::SpanContext const*)+0x4c2) [0x564e5454eb12]
 5: (rgw::cls::fifo::FIFO::push_entries(DoutPrefixProvider const*, std::deque<ceph::buffer::v15_2_0::list, std::allocator<ceph::buffer::v15_2_0::list> > const&, unsigned long, optional_yield)+0x605) [0x564e546cfa75]
 6: (rgw::cls::fifo::FIFO::push(DoutPrefixProvider const*, std::vector<ceph::buffer::v15_2_0::list, std::allocator<ceph::buffer::v15_2_0::list> > const&, optional_yield)+0x9fe) [0x564e546d20de]
 7: (rgw::cls::fifo::FIFO::push(DoutPrefixProvider const*, ceph::buffer::v15_2_0::list const&, optional_yield)+0x13c) [0x564e546d2e3c]
 8: radosgw-admin(+0x7ff3cb) [0x564e543a63cb]
 9: (RGWDataChangesLog::add_entry(DoutPrefixProvider const*, RGWBucketInfo const&, rgw::bucket_log_layout_generation const&, int, optional_yield)+0x8be) [0x564e543c0fbe]
 10: (add_datalog_entry(DoutPrefixProvider const*, RGWDataChangesLog*, RGWBucketInfo const&, unsigned int, optional_yield)+0x49) [0x564e5444ea79]
 11: (RGWRados::Bucket::UpdateIndex::complete(DoutPrefixProvider const*, long, unsigned long, unsigned long, unsigned long, std::chrono::time_point<ceph::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, ACLOwner const&, RGWObjCategory, std::__cxx11::list<rgw_obj_index_key, std::allocator<rgw_obj_index_key> >*, optional_yield, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const*, bool, bool)+0x2e3) [0x564e5447d603]
 12: (RGWRados::Object::Write::_do_write_meta(unsigned long, unsigned long, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&, bool, bool, void*, req_context const&, opentelemetry::v1::trace::SpanContext&, bool)+0x9f3) [0x564e5446a1d3]
 13: (RGWRados::Object::Write::write_meta(unsigned long, unsigned long, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&, req_context const&, opentelemetry::v1::trace::SpanContext&, bool)+0x55b) [0x564e5446b5ab]
 14: (rgw::putobj::AtomicObjectProcessor::complete(unsigned long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::chrono::time_point<ceph::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >*, std::chrono::time_point<ceph::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&, std::chrono::time_point<ceph::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, char const*, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const*, rgw_zone_set*, bool*, req_context const&, unsigned int)+0x3c6) [0x564e54451a36]
 15: (RGWRados::fetch_remote_obj(RGWObjectCtx&, rgw_user const&, req_info*, rgw_zone_id const&, rgw_obj const&, rgw_obj const&, RGWBucketInfo&, RGWBucketInfo*, std::optional<rgw_placement_rule>, std::chrono::time_point<ceph::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >*, std::chrono::time_point<ceph::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >*, std::chrono::time_point<ceph::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const*, std::chrono::time_point<ceph::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const*, bool, char const*, char const*, RGWRados::AttrsMod, bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&, RGWObjCategory, std::optional<unsigned long>, std::chrono::time_point<ceph::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*, void (*)(long, void*), void*, req_context const&, RGWFetchObjFilter*, bool, rgw_obj const&, rgw_zone_set_entry const&, rgw_zone_set*, std::optional<unsigned long>*)+0x1cd2) [0x564e544714a2]
 16: (RGWAsyncFetchRemoteObj::_send_request(DoutPrefixProvider const*)+0x722) [0x564e543aa5f2]
 17: (RGWAsyncRadosProcessor::RGWWQ::_process(RGWAsyncRadosRequest*, ThreadPool::TPHandle&)+0x31) [0x564e54385131]
 18: (ThreadPool::worker(ThreadPool::WorkThread*)+0x932) [0x7f502cadab22]
 19: /usr/lib/ceph/libceph-common.so.2(+0x260695) [0x7f502cad6695]
 20: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7f502b8ccb43]
 21: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7f502b95ea00]
2024-04-16T02:09:49.152 INFO:scrape:1 jobs: ['7657856']
2024-04-16T02:09:49.152 INFO:scrape:suites: ['frontend/beast', 'ignore-pg-availability', 'notify', 'omap_limits', 'overrides', 'realms/three-zones', 'rgw:multisite/{clusters', 'supported-random-distro$/{ubuntu_latest}', 'tasks/test_multi}']
2024-04-16T02:09:49.152 INFO:scrape:
2024-04-16T02:09:49.152 INFO:scrape:Failure: rgw multisite test failures
2024-04-16T02:09:49.152 INFO:scrape:1 jobs: ['7657857']
2024-04-16T02:09:49.152 INFO:scrape:suites: ['frontend/beast', 'ignore-pg-availability', 'notify', 'omap_limits', 'overrides', 'realms/two-zonegroup', 'rgw:multisite/{clusters', 'supported-random-distro$/{centos_latest}', 'tasks/test_multi}']
2024-04-16T02:09:49.153 INFO:scrape: