Found 68 jobs
Found 9 distinct failure reasons
Failure: "2021-09-29T19:12:35.408198+0000 mon.a (mon.0) 118 : cluster [WRN] Health check failed: Degraded data redundancy: 2/4 objects degraded (50.000%), 1 pg degraded (PG_DEGRADED)" in cluster log
2 jobs: ['6414117', '6414122']
suites intersection: ['frontend/beast', 'objectstore/filestore-xfs']
suites union: ['frontend/beast', 'ignore-pg-availability', 'install', 'objectstore/filestore-xfs', 'overrides', 'rgw/multifs/{clusters/fixed-2', 'rgw/thrash/{clusters/fixed-2', 'rgw_pool_type/ec', 'tasks/rgw_bucket_quota', 'thrasher/default', 'thrashosds-health', 'ubuntu_latest', 'ubuntu_latest}', 'workload/rgw_s3tests}']

Crash: Command failed (workunit test rgw/run-reshard.sh) on smithi018 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=1d16d7d64dce70cbd43ac42a52fbb2d8f24d0f23 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/run-reshard.sh'
ceph version 17.0.0-7955-g1d16d7d6 (1d16d7d64dce70cbd43ac42a52fbb2d8f24d0f23) quincy (dev)
 1: /lib64/libpthread.so.0(+0x12dc0) [0x7fd444d50dc0]
 2: gsignal()
 3: abort()
 4: /lib64/libc.so.6(+0x21bc9) [0x7fd4439a6bc9]
 5: /lib64/libc.so.6(+0x2fe96) [0x7fd4439b4e96]
 6: ceph-osd(+0x7c7695) [0x56057e352695]
 7: (PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0xe46) [0x56057e373666]
 8: (PrimaryLogPG::simple_opc_submit(std::unique_ptr<PrimaryLogPG::OpContext, std::default_delete<PrimaryLogPG::OpContext> >)+0x120) [0x56057e375750]
 9: (PrimaryLogPG::AwaitAsyncWork::react(PrimaryLogPG::DoSnapWork const&)+0x766) [0x56057e3b7666]
 10: (boost::statechart::simple_state<PrimaryLogPG::AwaitAsyncWork, PrimaryLogPG::Trimming, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x121) [0x56057e43ba31]
 11: (boost::statechart::state_machine<PrimaryLogPG::SnapTrimmer, PrimaryLogPG::NotTrimming, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x74) [0x56057e404654]
 12: (PrimaryLogPG::snap_trimmer(unsigned int)+0x17a) [0x56057e3695da]
 13: (ceph::osd::scheduler::PGSnapTrim::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x1f) [0x56057e5259af]
 14: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xac8) [0x56057e281e48]
 15: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x56057e970c64]
 16: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x56057e971754]
 17: /lib64/libpthread.so.0(+0x82de) [0x7fd444d462de]
 18: clone()
23 jobs: ['6414124', '6414120', '6414119', '6414087', '6414132', '6414083', '6414099', '6414081', '6414093', '6414091', '6414131', '6414134', '6414089', '6414115', '6414101', '6414128', '6414077', '6414097', '6414112', '6414111', '6414123', '6414109', '6414102']
suites intersection: ['centos_latest', 'clusters/fixed-2', 'frontend/beast', 'ignore-pg-availability', 'msgr-failures/few', 'overrides', 'ragweed', 'reshard', 'rgw/verify/{0-install', 's3tests-java', 's3tests}', 'tasks/{cls']
suites union: ['centos_latest', 'clusters/fixed-2', 'datacache/no_datacache', 'datacache/rgw-datacache', 'frontend/beast', 'ignore-pg-availability', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'overrides', 'proto/http', 'proto/https', 'ragweed', 'reshard', 'rgw/verify/{0-install', 'rgw_pool_type/ec', 'rgw_pool_type/ec-profile', 'rgw_pool_type/replicated', 's3tests-java', 's3tests}', 'sharding$/{default}', 'sharding$/{single}', 'striping$/{stripe-equals-chunk}', 'striping$/{stripe-greater-than-chunk}', 'tasks/{cls', 'validater/lockdep}', 'validater/valgrind}']

Crash: Command failed on smithi031 with status 11: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg deep-scrub 3.3f'
ceph version 17.0.0-7955-g1d16d7d6 (1d16d7d64dce70cbd43ac42a52fbb2d8f24d0f23) quincy (dev)
 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x153c0) [0x7f747daeb3c0]
 2: gsignal()
 3: abort()
 4: /lib/x86_64-linux-gnu/libc.so.6(+0x25729) [0x7f747d54f729]
 5: /lib/x86_64-linux-gnu/libc.so.6(+0x36f36) [0x7f747d560f36]
 6: ceph-osd(+0xdfa9b5) [0x557442d1b9b5]
 7: (PrimaryLogPG::finish_ctx(PrimaryLogPG::OpContext*, int, int)+0x1fd4) [0x557442d4d6d4]
 8: (PrimaryLogScrub::scrub_snapshot_metadata(ScrubMap&, std::map<hobject_t, std::pair<std::optional<unsigned int>, std::optional<unsigned int> >, std::less<hobject_t>, std::allocator<std::pair<hobject_t const, std::pair<std::optional<unsigned int>, std::optional<unsigned int> > > > > const&)+0x2856) [0x557442e86f46]
 9: (PgScrubber::scrub_compare_maps()+0x1da5) [0x557442e72325]
 10: (PgScrubber::maps_compare_n_cleanup()+0xd) [0x557442e728fd]
 11: (Scrub::WaitReplicas::react(Scrub::GotReplicas const&)+0x15d) [0x557442ea835d]
 12: (boost::statechart::simple_state<Scrub::WaitReplicas, Scrub::ActiveScrubbing, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x101) [0x557442eb1c41]
 13: (boost::statechart::state_machine<Scrub::ScrubMachine, Scrub::NotActive, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x172) [0x557442e77c12]
 14: (PgScrubber::send_scrub_resched(unsigned int)+0x117) [0x557442e66437]
 15: (PG::forward_scrub_event(void (ScrubPgIF::*)(unsigned int), unsigned int, std::basic_string_view<char, std::char_traits<char> >)+0x7c) [0x557442cd722c]
 16: (ceph::osd::scheduler::PGScrubResched::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x37) [0x557442f278a7]
 17: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x8aa) [0x557442c3745a]
 18: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x403) [0x5574433b2213]
 19: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x5574433b5034]
 20: /lib/x86_64-linux-gnu/libpthread.so.0(+0x9609) [0x7f747dadf609]
 21: clone()
11 jobs: ['6414075', '6414082', '6414105', '6414098', '6414074', '6414108', '6414068', '6414090', '6414114', '6414130', '6414078']
suites intersection: []
suites union: ['frontend/beast', 'hadoop/default', 'hadoop/v32', 'http', 'install', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'overrides', 'rgw/hadoop-s3a/{clusters/fixed-2', 'rgw/tempest/{clusters/fixed-1', 'rgw/thrash/{clusters/fixed-2', 'rgw/website/{clusters/fixed-2', 's3a-hadoop}', 'tasks/rgw_tempest', 'tasks/s3tests-website', 'thrasher/default', 'thrashosds-health', 'ubuntu_latest', 'ubuntu_latest}', 'workload/rgw_bucket_quota}', 'workload/rgw_multipart_upload}', 'workload/rgw_s3tests}', 'workload/rgw_user_quota}']

Dead: b'2021-09-30T06:47:03.448 DEBUG:teuthology.exit:Finished running handlers'
ceph version 17.0.0-7955-g1d16d7d6 (1d16d7d64dce70cbd43ac42a52fbb2d8f24d0f23) quincy (dev)
 1: /lib64/libpthread.so.0(+0x12dc0) [0x7f6bd4e69dc0]
 2: gsignal()
 3: abort()
 4: /lib64/libc.so.6(+0x21bc9) [0x7f6bd3abfbc9]
 5: /lib64/libc.so.6(+0x2fe96) [0x7f6bd3acde96]
 6: ceph-osd(+0x7c7695) [0x55d6edfd8695]
 7: (PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0xe46) [0x55d6edff9666]
 8: (PrimaryLogPG::simple_opc_submit(std::unique_ptr<PrimaryLogPG::OpContext, std::default_delete<PrimaryLogPG::OpContext> >)+0x120) [0x55d6edffb750]
 9: (PrimaryLogPG::AwaitAsyncWork::react(PrimaryLogPG::DoSnapWork const&)+0x766) [0x55d6ee03d666]
 10: (boost::statechart::simple_state<PrimaryLogPG::AwaitAsyncWork, PrimaryLogPG::Trimming, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x121) [0x55d6ee0c1a31]
 11: (boost::statechart::state_machine<PrimaryLogPG::SnapTrimmer, PrimaryLogPG::NotTrimming, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x74) [0x55d6ee08a654]
 12: (PrimaryLogPG::snap_trimmer(unsigned int)+0x17a) [0x55d6edfef5da]
 13: (ceph::osd::scheduler::PGSnapTrim::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x1f) [0x55d6ee1ab9af]
 14: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xac8) [0x55d6edf07e48]
 15: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x55d6ee5f6c64]
 16: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x55d6ee5f7754]
 17: /lib64/libpthread.so.0(+0x82de) [0x7f6bd4e5f2de]
 18: clone()
1 jobs: ['6414079']
suites: ['centos_latest', 'clusters/fixed-2', 'datacache/no_datacache', 'frontend/beast', 'ignore-pg-availability', 'msgr-failures/few', 'objectstore/filestore-xfs', 'overrides', 'proto/https', 'ragweed', 'reshard', 'rgw/verify/{0-install', 'rgw_pool_type/ec', 's3tests-java', 's3tests}', 'sharding$/{default}', 'striping$/{stripe-equals-chunk}', 'tasks/{cls', 'validater/valgrind}']

Failure: rgw multisite test failures
2 jobs: ['6414070', '6414106']
suites intersection: ['frontend/beast', 'ignore-pg-availability', 'omap_limits', 'overrides', 'rgw/multisite/{clusters', 'tasks/test_multi}']
suites union: ['frontend/beast', 'ignore-pg-availability', 'omap_limits', 'overrides', 'realms/three-zone-plus-pubsub', 'realms/two-zonegroup', 'rgw/multisite/{clusters', 'supported-random-distro$/{centos_8.stream}', 'supported-random-distro$/{centos_8}', 'tasks/test_multi}']

Dead: b'2021-09-30T06:43:27.929 DEBUG:teuthology.exit:Finished running handlers'
2 jobs: ['6414071', '6414085']
suites intersection: []
suites union: ['1-ceph-install/install', '2-kms/kmip', '3-rgw/rgw', '4-tests/{s3tests}', 'bluestore-bitmap', 'fixed-2', 'ignore-pg-availability', 'overrides', 'rgw/crypt/{0-cluster/fixed-1', 'rgw/notifications/{beast', 'supported-all-distro$/{centos_8}', 'supported-random-distro$/{centos_8.stream}}', 'tasks/{0-install', 'test_amqp', 'test_kafka', 'test_others}}']

Failure: Command failed (s3 tests against rgw) on smithi151 with status 1: "S3TEST_CONF=/home/ubuntu/cephtest/archive/s3-tests.client.0.conf BOTO_CONFIG=/home/ubuntu/cephtest/boto.cfg REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt /home/ubuntu/cephtest/s3-tests/virtualenv/bin/python -m nose -w /home/ubuntu/cephtest/s3-tests -v -a '!fails_on_rgw,!lifecycle_expiration,!fails_strict_rfc2616,!test_of_sts,!webidentity_test,!fails_with_subdomain'"
2 jobs: ['6414116', '6414104']
suites intersection: ['1-ceph-install/install', '3-rgw/rgw', '4-tests/{s3tests}', 'ignore-pg-availability', 'rgw/crypt/{0-cluster/fixed-1']
suites union: ['1-ceph-install/install', '2-kms/vault_kv', '2-kms/vault_old', '3-rgw/rgw', '4-tests/{s3tests}', 'ignore-pg-availability', 'rgw/crypt/{0-cluster/fixed-1', 'supported-random-distro$/{centos_8.stream}}', 'supported-random-distro$/{rhel_8}}']

Crash: "2021-09-29T21:24:42.870474+0000 mon.a (mon.0) 464 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
ceph version 17.0.0-7955-g1d16d7d6 (1d16d7d64dce70cbd43ac42a52fbb2d8f24d0f23) quincy (dev)
 1: /lib64/libpthread.so.0(+0x12b20) [0x7fbb195fdb20]
 2: gsignal()
 3: abort()
 4: /lib64/libc.so.6(+0x21c89) [0x7fbb18249c89]
 5: /lib64/libc.so.6(+0x2fa76) [0x7fbb18257a76]
 6: ceph-osd(+0x7c7695) [0x55756259d695]
 7: (PrimaryLogPG::finish_ctx(PrimaryLogPG::OpContext*, int, int)+0x14f5) [0x5575625c8245]
 8: (PrimaryLogScrub::scrub_snapshot_metadata(ScrubMap&, std::map<hobject_t, std::pair<std::optional<unsigned int>, std::optional<unsigned int> >, std::less<hobject_t>, std::allocator<std::pair<hobject_t const, std::pair<std::optional<unsigned int>, std::optional<unsigned int> > > > > const&)+0x3ada) [0x5575626dbdca]
 9: (PgScrubber::scrub_compare_maps()+0x1b97) [0x5575626bbe67]
 10: (PgScrubber::maps_compare_n_cleanup()+0xd) [0x5575626bc05d]
 11: (Scrub::WaitReplicas::react(Scrub::GotReplicas const&)+0x2ec) [0x5575626f92dc]
 12: (boost::statechart::simple_state<Scrub::WaitReplicas, Scrub::ActiveScrubbing, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0xe1) [0x5575627006d1]
 13: (boost::statechart::state_machine<Scrub::ScrubMachine, Scrub::NotActive, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x74) [0x5575626cca04]
 14: (PgScrubber::send_replica_maps_ready(unsigned int)+0x2fd) [0x5575626c363d]
 15: (PG::forward_scrub_event(void (ScrubPgIF::*)(unsigned int), unsigned int, std::basic_string_view<char, std::char_traits<char> >)+0x82) [0x557562563062]
 16: (ceph::osd::scheduler::PGScrubGotReplMaps::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x2f) [0x557562770ecf]
 17: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xac8) [0x5575624cce48]
 18: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x557562bbbc64]
 19: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x557562bbc754]
 20: /lib64/libpthread.so.0(+0x814a) [0x7fbb195f314a]
 21: clone()
1 jobs: ['6414076']
suites: ['cluster', 'rgw/tools/{centos_latest', 'tasks}']

Failure: 
6 jobs: ['6414086', '6414127', '6414096', '6414118', '6414107', '6414072']
suites intersection: ['frontend/beast', 'overrides', 'rgw/singleton/{all/radosgw-admin']
suites union: ['frontend/beast', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'overrides', 'rgw/singleton/{all/radosgw-admin', 'rgw_pool_type/ec', 'rgw_pool_type/ec-profile', 'rgw_pool_type/replicated', 'supported-random-distro$/{centos_8.stream}}', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{rhel_8}}', 'supported-random-distro$/{ubuntu_latest}}']