diff --git a/swh/loader/core/loader.py b/swh/loader/core/loader.py --- a/swh/loader/core/loader.py +++ b/swh/loader/core/loader.py @@ -11,6 +11,7 @@ from typing import Any, Dict, Iterable, Optional from swh.core.config import load_from_envvar +from swh.loader.exception import NotFound from swh.model.model import ( BaseContent, Content, @@ -331,6 +332,21 @@ ) self.storage.origin_visit_status_add([visit_status]) self.post_load() + except NotFound: + self.log.exception( + "Loading failure, updating to `not_found` status", + extra={"swh_task_args": args, "swh_task_kwargs": kwargs,}, + ) + visit_status = OriginVisitStatus( + origin=self.origin.url, + visit=self.visit.visit, + type=self.visit_type, + date=now(), + status="not_found", + snapshot=None, + ) + self.storage.origin_visit_status_add([visit_status]) + self.post_load(success=False) except Exception: status = "partial" if self.loaded_snapshot_id else "failed" self.log.exception( diff --git a/swh/loader/core/tests/test_loader.py b/swh/loader/core/tests/test_loader.py --- a/swh/loader/core/tests/test_loader.py +++ b/swh/loader/core/tests/test_loader.py @@ -8,6 +8,7 @@ import logging from swh.loader.core.loader import DEFAULT_CONFIG, BaseLoader, DVCSLoader +from swh.loader.exception import NotFound from swh.loader.tests import assert_last_visit_matches from swh.model.hashutil import hash_to_bytes from swh.model.model import Origin, OriginVisit, Snapshot @@ -135,6 +136,7 @@ def _check_load_failure(caplog, loader, exc_class, exc_text, status="partial"): """Check whether a failed load properly logged its exception, and that the snapshot didn't get referenced in storage""" + assert isinstance(loader, DVCSLoader) # was implicit so far for record in caplog.records: if record.levelname != "ERROR": continue @@ -213,3 +215,26 @@ _check_load_failure( caplog, loader, RuntimeError, "Failed to add snapshot!", status="failed" ) + + +class DummyDVCSLoaderNotFound(DummyDVCSLoader, BaseLoader): + """A loader which raises a not_found exception during the prepare method call + + """ + + def prepare(*args, **kwargs): + raise NotFound("Unknown origin!") + + def load_status(self): + return { + "status": "failed", + } + + +def test_loader_not_found(swh_config, caplog): + loader = DummyDVCSLoaderNotFound() + result = loader.load() + + assert result == {"status": "failed"} + + _check_load_failure(caplog, loader, NotFound, "Unknown origin!", status="not_found")