diff --git a/swh/indexer/indexer.py b/swh/indexer/indexer.py --- a/swh/indexer/indexer.py +++ b/swh/indexer/indexer.py @@ -133,6 +133,10 @@ USE_TOOLS = True + catch_exceptions = True + """Prevents exceptions in `index()` from raising too high. Set to False + in tests to properly catch all exceptions.""" + def __init__(self, config=None, **kw): """Prepare and check that the indexer is ready to run. @@ -374,6 +378,8 @@ self.results = results return self.next_step(results, task=next_step) except Exception: + if not self.catch_exceptions: + raise self.log.exception( 'Problem when reading contents metadata.') @@ -512,6 +518,8 @@ results, policy_update='update-dups') with_indexed_data = True except Exception: + if not self.catch_exceptions: + raise self.log.exception( 'Problem when computing metadata.') finally: @@ -607,6 +615,8 @@ if res: # If no results, skip it results.append(res) except Exception: + if not self.catch_exceptions: + raise self.log.exception( 'Problem when processing origin %s', origin['id']) @@ -651,6 +661,8 @@ if res: # If no results, skip it results.append(res) except Exception: + if not self.catch_exceptions: + raise self.log.exception( 'Problem when processing revision') self.persist_index_computations(results, policy_update) diff --git a/swh/indexer/tests/test_ctags.py b/swh/indexer/tests/test_ctags.py --- a/swh/indexer/tests/test_ctags.py +++ b/swh/indexer/tests/test_ctags.py @@ -116,6 +116,7 @@ def setUp(self): super().setUp() self.indexer = CtagsIndexer(config=CONFIG) + self.indexer.catch_exceptions = False self.idx_storage = self.indexer.idx_storage fill_storage(self.indexer.storage) fill_obj_storage(self.indexer.objstorage) diff --git a/swh/indexer/tests/test_fossology_license.py b/swh/indexer/tests/test_fossology_license.py --- a/swh/indexer/tests/test_fossology_license.py +++ b/swh/indexer/tests/test_fossology_license.py @@ -89,6 +89,7 @@ fossology_license.compute_license = mock_compute_license self.indexer = FossologyLicenseIndexer(CONFIG) + self.indexer.catch_exceptions = False self.idx_storage = self.indexer.idx_storage fill_storage(self.indexer.storage) fill_obj_storage(self.indexer.objstorage) @@ -138,6 +139,7 @@ fossology_license.compute_license = mock_compute_license self.indexer = FossologyLicenseRangeIndexer(config=RANGE_CONFIG) + self.indexer.catch_exceptions = False fill_storage(self.indexer.storage) fill_obj_storage(self.indexer.objstorage) diff --git a/swh/indexer/tests/test_language.py b/swh/indexer/tests/test_language.py --- a/swh/indexer/tests/test_language.py +++ b/swh/indexer/tests/test_language.py @@ -59,6 +59,7 @@ def setUp(self): self.indexer = LanguageIndexer(config=CONFIG) + self.indexer.catch_exceptions = False fill_storage(self.indexer.storage) fill_obj_storage(self.indexer.objstorage) diff --git a/swh/indexer/tests/test_mimetype.py b/swh/indexer/tests/test_mimetype.py --- a/swh/indexer/tests/test_mimetype.py +++ b/swh/indexer/tests/test_mimetype.py @@ -57,6 +57,7 @@ def setUp(self): self.indexer = MimetypeIndexer(config=CONFIG) + self.indexer.catch_exceptions = False self.idx_storage = self.indexer.idx_storage fill_storage(self.indexer.storage) fill_obj_storage(self.indexer.objstorage) @@ -106,6 +107,7 @@ def setUp(self): super().setUp() self.indexer = MimetypeRangeIndexer(config=RANGE_CONFIG) + self.indexer.catch_exceptions = False fill_storage(self.indexer.storage) fill_obj_storage(self.indexer.objstorage) diff --git a/swh/indexer/tests/test_origin_head.py b/swh/indexer/tests/test_origin_head.py --- a/swh/indexer/tests/test_origin_head.py +++ b/swh/indexer/tests/test_origin_head.py @@ -38,6 +38,7 @@ class OriginHead(unittest.TestCase): def setUp(self): self.indexer = OriginHeadTestIndexer() + self.indexer.catch_exceptions = False fill_storage(self.indexer.storage) def _get_origin_id(self, type_, url):