diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py --- a/swh/loader/package/loader.py +++ b/swh/loader/package/loader.py @@ -609,12 +609,20 @@ errors=[str(e)], ) + errors = [] + # Get the metadata of each version's package - packages_info: List[Tuple[str, TPackageInfo]] = [ - (branch_name, p_info) - for version in versions - for (branch_name, p_info) in self.get_package_info(version) - ] + packages_info: List[Tuple[str, TPackageInfo]] = [] + for version in versions: + try: + for branch_name, p_info in self.get_package_info(version): + packages_info.append((branch_name, p_info)) + except Exception as e: + load_exceptions.append(e) + sentry_sdk.capture_exception(e) + error = f"Failed to get package info for version {version} of {self.origin.url}" + logger.exception(error) + errors.append(f"{error}: {e}") # Compute the ExtID of each of these packages known_extids = self._get_known_extids([p_info for (_, p_info) in packages_info]) @@ -630,7 +638,7 @@ tmp_releases: Dict[str, List[Tuple[str, Sha1Git]]] = { version: [] for version in versions } - errors = [] + for (branch_name, p_info) in packages_info: logger.debug("package_info: %s", p_info) diff --git a/swh/loader/package/tests/test_loader.py b/swh/loader/package/tests/test_loader.py --- a/swh/loader/package/tests/test_loader.py +++ b/swh/loader/package/tests/test_loader.py @@ -610,3 +610,28 @@ sentry_tags.get(SENTRY_VISIT_TYPE_TAG_NAME) == StubPackageLoaderWithError.visit_type ) + + +class StubPackageLoaderWithPackageInfoFailure(StubPackageLoader): + def get_package_info(self, version): + if version == "v2.0": + raise Exception("Error when getting package info") + else: + return super().get_package_info(version) + + +def test_loader_origin_with_package_info_failure(swh_storage, requests_mock_datadir): + + loader = StubPackageLoaderWithPackageInfoFailure(swh_storage, ORIGIN_URL) + + assert loader.load() == { + "snapshot_id": "b4cce7081d661fb7f4d7a1db96e8044b752eb0b0", + "status": "eventful", + } + + assert loader.load_status() == {"status": "eventful"} + assert loader.visit_status() == "partial" + + assert set(loader.last_snapshot().branches.keys()) == { + f"branch-v{i}.0".encode() for i in (1, 3, 4) + }