Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/loader.py
Show All 12 Lines | |||||
from swh.core import config | from swh.core import config | ||||
from swh.model.model import ( | from swh.model.model import ( | ||||
BaseContent, | BaseContent, | ||||
Content, | Content, | ||||
SkippedContent, | SkippedContent, | ||||
Directory, | Directory, | ||||
Origin, | Origin, | ||||
OriginVisitStatus, | |||||
Revision, | Revision, | ||||
Release, | Release, | ||||
Sha1Git, | Sha1Git, | ||||
Snapshot, | Snapshot, | ||||
) | ) | ||||
from swh.storage import get_storage | from swh.storage import get_storage | ||||
from swh.storage.utils import now | |||||
class BaseLoader(config.SWHConfig, metaclass=ABCMeta): | class BaseLoader(config.SWHConfig, metaclass=ABCMeta): | ||||
"""Mixin base class for loader. | """Mixin base class for loader. | ||||
To use this class, you must: | To use this class, you must: | ||||
- inherit from this class | - inherit from this class | ||||
▲ Show 20 Lines • Show All 264 Lines • ▼ Show 20 Lines | def load(self, *args, **kwargs) -> Dict[str, str]: | ||||
while True: | while True: | ||||
more_data_to_fetch = self.fetch_data() | more_data_to_fetch = self.fetch_data() | ||||
self.store_data() | self.store_data() | ||||
if not more_data_to_fetch: | if not more_data_to_fetch: | ||||
break | break | ||||
self.store_metadata() | self.store_metadata() | ||||
self.storage.origin_visit_update( | visit_status = OriginVisitStatus( | ||||
self.origin.url, | origin=self.origin.url, | ||||
self.visit.visit, | visit=self.visit.visit, | ||||
self.visit_status(), | date=now(), | ||||
status=self.visit_status(), | |||||
snapshot=self.loaded_snapshot_id, | snapshot=self.loaded_snapshot_id, | ||||
) | ) | ||||
self.storage.origin_visit_status_add([visit_status]) | |||||
self.post_load() | self.post_load() | ||||
except Exception: | except Exception: | ||||
self.log.exception( | self.log.exception( | ||||
"Loading failure, updating to `partial` status", | "Loading failure, updating to `partial` status", | ||||
extra={"swh_task_args": args, "swh_task_kwargs": kwargs,}, | extra={"swh_task_args": args, "swh_task_kwargs": kwargs,}, | ||||
) | ) | ||||
self.storage.origin_visit_update( | visit_status = OriginVisitStatus( | ||||
self.origin.url, | origin=self.origin.url, | ||||
self.visit.visit, | visit=self.visit.visit, | ||||
"partial", | date=now(), | ||||
status="partial", | |||||
snapshot=self.loaded_snapshot_id, | snapshot=self.loaded_snapshot_id, | ||||
) | ) | ||||
self.storage.origin_visit_status_add([visit_status]) | |||||
self.post_load(success=False) | self.post_load(success=False) | ||||
return {"status": "failed"} | return {"status": "failed"} | ||||
finally: | finally: | ||||
self.flush() | self.flush() | ||||
self.cleanup() | self.cleanup() | ||||
return self.load_status() | return self.load_status() | ||||
▲ Show 20 Lines • Show All 85 Lines • Show Last 20 Lines |