diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py --- a/swh/loader/package/loader.py +++ b/swh/loader/package/loader.py @@ -160,6 +160,18 @@ """ super().__init__(storage=storage, origin_url=url, **kwargs) + self.status_load = "" + self.status_visit = "" + + def load_status(self) -> Dict[str, str]: + """Detailed loading status.""" + return { + "status": self.status_load, + } + + def visit_status(self) -> str: + """Detailed visit status.""" + return self.status_visit def get_versions(self) -> Sequence[str]: """Return the list of all published package versions. @@ -448,6 +460,8 @@ - return the task's status """ + self.status_load = status_load + self.status_visit = status_visit self.storage.flush() snapshot_id: Optional[bytes] = None @@ -524,8 +538,8 @@ Software Heritage archive """ - status_load = "uneventful" # either: eventful, uneventful, failed - status_visit = "full" # see swh.model.model.OriginVisitStatus + self.status_load = "uneventful" # either: eventful, uneventful, failed + self.status_visit = "full" # see swh.model.model.OriginVisitStatus snapshot = None failed_branches: List[str] = [] @@ -549,6 +563,7 @@ "Failed to initialize origin_visit for %s", self.origin.url ) sentry_sdk.capture_exception(e) + self.status_load = self.status_visit = "failed" return {"status": "failed"} # Get the previous snapshot for this origin. It is then used to see which @@ -655,7 +670,7 @@ p_info, release_id, directory_id ) self.storage.flush() - status_load = "eventful" + self.status_load = "eventful" except Exception as e: self.storage.clear_buffers() load_exceptions.append(e) @@ -710,7 +725,7 @@ tmp_releases[p_info.version].append((branch_name, release_id)) if load_exceptions: - status_visit = "partial" + self.status_visit = "partial" if not tmp_releases: # We could not load any releases; fail completely @@ -741,8 +756,8 @@ logger.exception(error) errors.append(f"{error}: {e}") sentry_sdk.capture_exception(e) - status_visit = "failed" - status_load = "failed" + self.status_visit = "failed" + self.status_load = "failed" if snapshot: try: @@ -755,8 +770,8 @@ logger.exception(error) errors.append(f"{error}: {e}") sentry_sdk.capture_exception(e) - status_visit = "partial" - status_load = "failed" + self.status_visit = "partial" + self.status_load = "failed" try: metadata_objects = self.build_extrinsic_origin_metadata() @@ -766,18 +781,18 @@ logger.exception(error) errors.append(f"{error}: {e}") sentry_sdk.capture_exception(e) - status_visit = "partial" - status_load = "failed" + self.status_visit = "partial" + self.status_load = "failed" - if status_load != "failed": + if self.status_load != "failed": self._load_extids(new_extids) return self.finalize_visit( snapshot=snapshot, visit=visit, failed_branches=failed_branches, - status_visit=status_visit, - status_load=status_load, + status_visit=self.status_visit, + status_load=self.status_load, errors=errors, ) diff --git a/swh/loader/package/tests/test_loader.py b/swh/loader/package/tests/test_loader.py --- a/swh/loader/package/tests/test_loader.py +++ b/swh/loader/package/tests/test_loader.py @@ -115,6 +115,9 @@ "status": "eventful", } + assert loader.load_status() == {"status": "eventful"} + assert loader.visit_status() == "full" + assert set(loader.last_snapshot().branches.keys()) == { f"branch-{version}".encode() for version in loader.get_versions() } @@ -128,11 +131,17 @@ actual_load_status = loader.load() assert actual_load_status == {"status": "failed"} + assert loader.load_status() == {"status": "failed"} + assert loader.visit_status() == "failed" + loader.storage = FakeStorage2() actual_load_status2 = loader.load() assert actual_load_status2 == {"status": "failed"} + assert loader.load_status() == {"status": "failed"} + assert loader.visit_status() == "failed" + def test_resolve_object_from_extids() -> None: storage = get_storage("memory") @@ -313,6 +322,9 @@ loader.load() + assert loader.load_status() == {"status": "eventful"} + assert loader.visit_status() == "full" + assert loader._load_release.mock_calls == [ # type: ignore # v1.0: not loaded because there is already its (extid_type, extid, rel) # in the storage. @@ -483,6 +495,9 @@ loader.load() + assert loader.load_status() == {"status": "eventful"} + assert loader.visit_status() == "full" + assert len(caplog.records) == 1 (record,) = caplog.records assert record.levelname == "ERROR" @@ -587,6 +602,8 @@ origin_url = ORIGIN_URL loader = StubPackageLoaderWithError(swh_storage, origin_url) loader.load() + assert loader.load_status() == {"status": "failed"} + assert loader.visit_status() == "failed" sentry_tags = sentry_events[0]["tags"] assert sentry_tags.get(SENTRY_ORIGIN_URL_TAG_NAME) == origin_url assert (