2021-09-22T11:55:53.915 INFO:root:teuthology version: 1.1.0-d3a488f5
2021-09-22T11:55:53.917 INFO:teuthology.results:subset: '1/2'
2021-09-22T11:55:53.917 INFO:teuthology.results:seed: '7857'
2021-09-22T11:57:04.493 INFO:root:teuthology version: 1.1.0-d3a488f5
2021-09-22T11:57:04.496 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish...
2021-09-22T14:49:15.946 INFO:teuthology.results:Tests finished! gathering results...
2021-09-22T14:49:20.825 INFO:scrape:Found 68 jobs
2021-09-22T14:49:41.633 INFO:scrape:Found 9 distinct failure reasons
2021-09-22T14:49:41.635 INFO:scrape:Crash: Command failed on smithi124 with status 1: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin -n client.0 user rm --uid ragweed.client.0 --purge-data'
2021-09-22T14:49:41.635 INFO:scrape:ceph version 17.0.0-7814-gfba2d3f9 (fba2d3f907fa575ca22181f3aac39e6a4fd661de) quincy (dev)
 1: /lib64/libpthread.so.0(+0x12dc0) [0x7ff3b5278dc0]
 2: gsignal()
 3: abort()
 4: /lib64/libc.so.6(+0x21bc9) [0x7ff3b3ecebc9]
 5: /lib64/libc.so.6(+0x2fe96) [0x7ff3b3edce96]
 6: ceph-osd(+0x7c88e5) [0x5614d53d98e5]
 7: (PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0xe46) [0x5614d53fa8b6]
 8: (PrimaryLogPG::simple_opc_submit(std::unique_ptr<PrimaryLogPG::OpContext, std::default_delete<PrimaryLogPG::OpContext> >)+0x120) [0x5614d53fc9a0]
 9: (PrimaryLogPG::AwaitAsyncWork::react(PrimaryLogPG::DoSnapWork const&)+0x766) [0x5614d543e896]
 10: (boost::statechart::simple_state<PrimaryLogPG::AwaitAsyncWork, PrimaryLogPG::Trimming, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x121) [0x5614d54c2a21]
 11: (boost::statechart::state_machine<PrimaryLogPG::SnapTrimmer, PrimaryLogPG::NotTrimming, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x74) [0x5614d548b624]
 12: (PrimaryLogPG::snap_trimmer(unsigned int)+0x17a) [0x5614d53f082a]
 13: (ceph::osd::scheduler::PGSnapTrim::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x1f) [0x5614d555517f]
 14: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xac8) [0x5614d5308ff8]
 15: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x5614d59ebe24]
 16: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x5614d59ec914]
 17: /lib64/libpthread.so.0(+0x82de) [0x7ff3b526e2de]
 18: clone()
2021-09-22T14:49:41.636 INFO:scrape:22 jobs: ['6402556', '6402559', '6402555', '6402537', '6402525', '6402547', '6402567', '6402527', '6402533', '6402548', '6402538', '6402570', '6402551', '6402535', '6402545', '6402560', '6402529', '6402568', '6402513', '6402519', '6402564', '6402517']
2021-09-22T14:49:41.636 INFO:scrape:suites intersection: ['centos_latest', 'clusters/fixed-2', 'frontend/beast', 'ignore-pg-availability', 'msgr-failures/few', 'overrides', 'ragweed', 'reshard', 'rgw/verify/{0-install', 's3tests-java', 's3tests}', 'tasks/{cls']
2021-09-22T14:49:41.637 INFO:scrape:suites union: ['centos_latest', 'clusters/fixed-2', 'datacache/no_datacache', 'datacache/rgw-datacache', 'frontend/beast', 'ignore-pg-availability', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'overrides', 'proto/http', 'proto/https', 'ragweed', 'reshard', 'rgw/verify/{0-install', 'rgw_pool_type/ec', 'rgw_pool_type/ec-profile', 'rgw_pool_type/replicated', 's3tests-java', 's3tests}', 'sharding$/{default}', 'sharding$/{single}', 'striping$/{stripe-equals-chunk}', 'striping$/{stripe-greater-than-chunk}', 'tasks/{cls', 'validater/lockdep}', 'validater/valgrind}']
2021-09-22T14:49:41.637 INFO:scrape:
2021-09-22T14:49:41.638 INFO:scrape:Failure: No module named 'tasks'
2021-09-22T14:49:41.638 INFO:scrape:6 jobs: ['6402510', '6402543', '6402507', '6402526', '6402503', '6402516']
2021-09-22T14:49:41.639 INFO:scrape:suites intersection: []
2021-09-22T14:49:41.639 INFO:scrape:suites union: ['1-ceph-install/install', '2-kms/barbican', '3-rgw/rgw', '4-tests/{s3tests}', 'bluestore-bitmap', 'fixed-2', 'frontend/beast', 'ignore-pg-availability', 'install', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'overrides', 'rgw/crypt/{0-cluster/fixed-1', 'rgw/multifs/{clusters/fixed-2', 'rgw/notifications/{beast', 'rgw/singleton/{all/radosgw-admin', 'rgw/tempest/{clusters/fixed-1', 'rgw/thrash/{clusters/fixed-2', 'rgw_pool_type/ec', 'rgw_pool_type/ec-profile', 'supported-all-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}}', 'supported-random-distro$/{ubuntu_latest}}', 'tasks/rgw_multipart_upload', 'tasks/rgw_tempest', 'tasks/{0-install', 'test_amqp', 'test_kafka', 'test_others}}', 'thrasher/default', 'thrashosds-health', 'ubuntu_latest', 'ubuntu_latest}', 'workload/rgw_s3tests}']
2021-09-22T14:49:41.639 INFO:scrape:
2021-09-22T14:49:41.640 INFO:scrape:Failure: rgw multisite test failures
2021-09-22T14:49:41.640 INFO:scrape:2 jobs: ['6402542', '6402506']
2021-09-22T14:49:41.641 INFO:scrape:suites intersection: ['frontend/beast', 'ignore-pg-availability', 'omap_limits', 'overrides', 'rgw/multisite/{clusters', 'tasks/test_multi}']
2021-09-22T14:49:41.641 INFO:scrape:suites union: ['frontend/beast', 'ignore-pg-availability', 'omap_limits', 'overrides', 'realms/three-zone-plus-pubsub', 'realms/two-zonegroup', 'rgw/multisite/{clusters', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}', 'tasks/test_multi}']
2021-09-22T14:49:41.642 INFO:scrape:
2021-09-22T14:49:41.642 INFO:scrape:Failure: reached maximum tries (90) after waiting for 540 seconds
2021-09-22T14:49:41.643 INFO:scrape:1 jobs: ['6402523']
2021-09-22T14:49:41.643 INFO:scrape:suites: ['centos_latest', 'clusters/fixed-2', 'datacache/rgw-datacache', 'frontend/beast', 'ignore-pg-availability', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'overrides', 'proto/http', 'ragweed', 'reshard', 'rgw/verify/{0-install', 'rgw_pool_type/ec', 's3tests-java', 's3tests}', 'sharding$/{default}', 'striping$/{stripe-greater-than-chunk}', 'tasks/{cls', 'validater/lockdep}']
2021-09-22T14:49:41.644 INFO:scrape:
2021-09-22T14:49:41.644 INFO:scrape:Failure: Command failed (s3 tests against rgw) on smithi081 with status 1: "S3TEST_CONF=/home/ubuntu/cephtest/archive/s3-tests.client.0.conf BOTO_CONFIG=/home/ubuntu/cephtest/boto.cfg REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt /home/ubuntu/cephtest/s3-tests/virtualenv/bin/python -m nose -w /home/ubuntu/cephtest/s3-tests -v -a '!fails_on_rgw,!lifecycle_expiration,!fails_strict_rfc2616,!test_of_sts,!webidentity_test,!fails_with_subdomain'"
2021-09-22T14:49:41.644 INFO:scrape:3 jobs: ['6402562', '6402540', '6402552']
2021-09-22T14:49:41.645 INFO:scrape:suites intersection: ['1-ceph-install/install', '3-rgw/rgw', '4-tests/{s3tests}', 'ignore-pg-availability', 'rgw/crypt/{0-cluster/fixed-1']
2021-09-22T14:49:41.645 INFO:scrape:suites union: ['1-ceph-install/install', '2-kms/vault_kv', '2-kms/vault_old', '2-kms/vault_transit', '3-rgw/rgw', '4-tests/{s3tests}', 'ignore-pg-availability', 'rgw/crypt/{0-cluster/fixed-1', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{rhel_8}}']
2021-09-22T14:49:41.646 INFO:scrape:
2021-09-22T14:49:41.646 INFO:scrape:Crash: "2021-09-22T12:21:54.282434+0000 mon.a (mon.0) 237 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
2021-09-22T14:49:41.647 INFO:scrape:ceph version 17.0.0-7814-gfba2d3f9 (fba2d3f907fa575ca22181f3aac39e6a4fd661de) quincy (dev)
 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x153c0) [0x7f6e9da363c0]
 2: gsignal()
 3: abort()
 4: /lib/x86_64-linux-gnu/libc.so.6(+0x25729) [0x7f6e9d49a729]
 5: /lib/x86_64-linux-gnu/libc.so.6(+0x36f36) [0x7f6e9d4abf36]
 6: ceph-osd(+0xdfb845) [0x56249aac6845]
 7: (PrimaryLogPG::finish_ctx(PrimaryLogPG::OpContext*, int, int)+0x1fd4) [0x56249aaf85b4]
 8: (PrimaryLogScrub::scrub_snapshot_metadata(ScrubMap&, std::map<hobject_t, std::pair<std::optional<unsigned int>, std::optional<unsigned int> >, std::less<hobject_t>, std::allocator<std::pair<hobject_t const, std::pair<std::optional<unsigned int>, std::optional<unsigned int> > > > > const&)+0x2856) [0x56249adcbc96]
 9: (PgScrubber::scrub_compare_maps()+0x1da5) [0x56249adb0895]
 10: (PgScrubber::maps_compare_n_cleanup()+0xd) [0x56249adb0e6d]
 11: (Scrub::WaitReplicas::react(Scrub::GotReplicas const&)+0x15d) [0x56249adbc99d]
 12: (boost::statechart::simple_state<Scrub::WaitReplicas, Scrub::ActiveScrubbing, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x101) [0x56249adc5ca1]
 13: (boost::statechart::state_machine<Scrub::ScrubMachine, Scrub::NotActive, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x74) [0x56249adb5b74]
 14: (PgScrubber::send_replica_maps_ready(unsigned int)+0x117) [0x56249ada5cb7]
 15: (PG::forward_scrub_event(void (ScrubPgIF::*)(unsigned int), unsigned int, std::basic_string_view<char, std::char_traits<char> >)+0x7c) [0x56249aa820bc]
 16: (ceph::osd::scheduler::PGScrubGotReplMaps::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x2f) [0x56249ac6edff]
 17: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x8aa) [0x56249a9e19da]
 18: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x403) [0x56249b14ebf3]
 19: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x56249b151a14]
 20: /lib/x86_64-linux-gnu/libpthread.so.0(+0x9609) [0x7f6e9da2a609]
 21: clone()
2021-09-22T14:49:41.647 INFO:scrape:10 jobs: ['6402541', '6402504', '6402558', '6402514', '6402534', '6402544', '6402518', '6402550', '6402566', '6402511']
2021-09-22T14:49:41.648 INFO:scrape:suites intersection: []
2021-09-22T14:49:41.648 INFO:scrape:suites union: ['frontend/beast', 'hadoop/default', 'hadoop/v32', 'http', 'install', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'overrides', 'rgw/hadoop-s3a/{clusters/fixed-2', 'rgw/thrash/{clusters/fixed-2', 'rgw/website/{clusters/fixed-2', 's3a-hadoop}', 'tasks/s3tests-website', 'thrasher/default', 'thrashosds-health', 'ubuntu_latest', 'ubuntu_latest}', 'workload/rgw_bucket_quota}', 'workload/rgw_multipart_upload}', 'workload/rgw_s3tests}', 'workload/rgw_user_quota}']
2021-09-22T14:49:41.649 INFO:scrape:
2021-09-22T14:49:41.649 INFO:scrape:Crash: Command failed on smithi109 with status 1: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin -n client.0 user rm --uid ragweed.client.0 --purge-data'
2021-09-22T14:49:41.649 INFO:scrape:ceph version 17.0.0-7814-gfba2d3f9 (fba2d3f907fa575ca22181f3aac39e6a4fd661de) quincy (dev)
 1: /lib64/libpthread.so.0(+0x12dc0) [0x7f19118d8dc0]
 2: gsignal()
 3: abort()
 4: /lib64/libc.so.6(+0x21bc9) [0x7f191052ebc9]
 5: /lib64/libc.so.6(+0x2fe96) [0x7f191053ce96]
 6: ceph-osd(+0x7c88e5) [0x5593e60028e5]
 7: (PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0xe46) [0x5593e60238b6]
 8: (PrimaryLogPG::simple_opc_submit(std::unique_ptr<PrimaryLogPG::OpContext, std::default_delete<PrimaryLogPG::OpContext> >)+0x120) [0x5593e60259a0]
 9: (PrimaryLogPG::AwaitAsyncWork::react(PrimaryLogPG::DoSnapWork const&)+0x766) [0x5593e6067896]
 10: (boost::statechart::simple_state<PrimaryLogPG::AwaitAsyncWork, PrimaryLogPG::Trimming, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x121) [0x5593e60eba21]
 11: (boost::statechart::state_machine<PrimaryLogPG::SnapTrimmer, PrimaryLogPG::NotTrimming, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x74) [0x5593e60b4624]
 12: (PrimaryLogPG::snap_trimmer(unsigned int)+0x17a) [0x5593e601982a]
 13: (ceph::osd::scheduler::PGSnapTrim::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x1f) [0x5593e617e17f]
 14: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xac8) [0x5593e5f31ff8]
 15: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x5593e6614e24]
 16: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x5593e6615914]
 17: /lib64/libpthread.so.0(+0x82de) [0x7f19118ce2de]
 18: clone()
2021-09-22T14:49:41.650 INFO:scrape:1 jobs: ['6402515']
2021-09-22T14:49:41.650 INFO:scrape:suites: ['centos_latest', 'clusters/fixed-2', 'datacache/no_datacache', 'frontend/beast', 'ignore-pg-availability', 'msgr-failures/few', 'objectstore/filestore-xfs', 'overrides', 'proto/https', 'ragweed', 'reshard', 'rgw/verify/{0-install', 'rgw_pool_type/ec', 's3tests-java', 's3tests}', 'sharding$/{default}', 'striping$/{stripe-greater-than-chunk}', 'tasks/{cls', 'validater/valgrind}']
2021-09-22T14:49:41.651 INFO:scrape:
2021-09-22T14:49:41.651 INFO:scrape:Failure: 
2021-09-22T14:49:41.652 INFO:scrape:5 jobs: ['6402532', '6402522', '6402554', '6402508', '6402563']
2021-09-22T14:49:41.652 INFO:scrape:suites intersection: ['frontend/beast', 'overrides', 'rgw/singleton/{all/radosgw-admin']
2021-09-22T14:49:41.653 INFO:scrape:suites union: ['frontend/beast', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'overrides', 'rgw/singleton/{all/radosgw-admin', 'rgw_pool_type/ec', 'rgw_pool_type/ec-profile', 'rgw_pool_type/replicated', 'supported-random-distro$/{centos_8.stream}}', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{rhel_8}}']
2021-09-22T14:49:41.653 INFO:scrape:
2021-09-22T14:49:41.654 INFO:scrape:Crash: "2021-09-22T14:39:53.989517+0000 mon.a (mon.0) 468 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
2021-09-22T14:49:41.654 INFO:scrape:ceph version 17.0.0-7814-gfba2d3f9 (fba2d3f907fa575ca22181f3aac39e6a4fd661de) quincy (dev)
 1: /lib64/libpthread.so.0(+0x12b20) [0x7f156fd55b20]
 2: gsignal()
 3: abort()
 4: /lib64/libc.so.6(+0x21c89) [0x7f156e9a1c89]
 5: /lib64/libc.so.6(+0x2fa76) [0x7f156e9afa76]
 6: ceph-osd(+0x7c88e5) [0x56209ea408e5]
 7: (PrimaryLogPG::finish_ctx(PrimaryLogPG::OpContext*, int, int)+0x14f5) [0x56209ea6b495]
 8: (PrimaryLogScrub::scrub_snapshot_metadata(ScrubMap&, std::map<hobject_t, std::pair<std::optional<unsigned int>, std::optional<unsigned int> >, std::less<hobject_t>, std::allocator<std::pair<hobject_t const, std::pair<std::optional<unsigned int>, std::optional<unsigned int> > > > > const&)+0x3ada) [0x56209ecfcd8a]
 9: (PgScrubber::scrub_compare_maps()+0x1b97) [0x56209ecd6bc7]
 10: (PgScrubber::maps_compare_n_cleanup()+0xd) [0x56209ecd6dbd]
 11: (Scrub::WaitReplicas::react(Scrub::GotReplicas const&)+0x2ec) [0x56209eceea7c]
 12: (boost::statechart::simple_state<Scrub::WaitReplicas, Scrub::ActiveScrubbing, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0xe1) [0x56209ecf58d1]
 13: (boost::statechart::state_machine<Scrub::ScrubMachine, Scrub::NotActive, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x74) [0x56209ece7664]
 14: (PgScrubber::send_replica_maps_ready(unsigned int)+0x2fd) [0x56209ecde39d]
 15: (PG::forward_scrub_event(void (ScrubPgIF::*)(unsigned int), unsigned int, std::basic_string_view<char, std::char_traits<char> >)+0x82) [0x56209ea06342]
 16: (ceph::osd::scheduler::PGScrubGotReplMaps::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x2f) [0x56209ebbc69f]
 17: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xac8) [0x56209e96fff8]
 18: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x56209f052e24]
 19: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x56209f053914]
 20: /lib64/libpthread.so.0(+0x814a) [0x7f156fd4b14a]
 21: clone()
2021-09-22T14:49:41.655 INFO:scrape:1 jobs: ['6402512']
2021-09-22T14:49:41.655 INFO:scrape:suites: ['cluster', 'rgw/tools/{centos_latest', 'tasks}']
2021-09-22T14:49:41.655 INFO:scrape: