Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/loader.py
Show First 20 Lines • Show All 326 Lines • ▼ Show 20 Lines | def load(self) -> Dict: | ||||
information from step 0., generate a snapshot and load it into the | information from step 0., generate a snapshot and load it into the | ||||
Software Heritage archive | Software Heritage archive | ||||
""" | """ | ||||
status_load = "uneventful" # either: eventful, uneventful, failed | status_load = "uneventful" # either: eventful, uneventful, failed | ||||
status_visit = "full" # either: partial, full | status_visit = "full" # either: partial, full | ||||
tmp_revisions = {} # type: Dict[str, List] | tmp_revisions = {} # type: Dict[str, List] | ||||
snapshot = None | snapshot = None | ||||
failed_branches: List[str] = [] | |||||
def finalize_visit() -> Dict[str, Any]: | def finalize_visit() -> Dict[str, Any]: | ||||
"""Finalize the visit: | """Finalize the visit: | ||||
- flush eventual unflushed data to storage | - flush eventual unflushed data to storage | ||||
- update origin visit's status | - update origin visit's status | ||||
- return the task's status | - return the task's status | ||||
Show All 12 Lines | def load(self) -> Dict: | ||||
snapshot=snapshot_id, | snapshot=snapshot_id, | ||||
) | ) | ||||
self.storage.origin_visit_status_add([visit_status]) | self.storage.origin_visit_status_add([visit_status]) | ||||
result: Dict[str, Any] = { | result: Dict[str, Any] = { | ||||
"status": status_load, | "status": status_load, | ||||
} | } | ||||
if snapshot_id: | if snapshot_id: | ||||
result["snapshot_id"] = hash_to_hex(snapshot_id) | result["snapshot_id"] = hash_to_hex(snapshot_id) | ||||
if failed_branches: | |||||
logger.warning( | |||||
"%d failed branches: %s", len(failed_branches), failed_branches, | |||||
) | |||||
return result | return result | ||||
vlorentz: that's a weird way to put it imo. What about this?
```
logger.warning(
"%d failed branches… | |||||
Done Inline Actionsyeah, i'm biased. ardumont: yeah, i'm biased.
I did it so one does not have to scroll too much backwards (if the list of… | |||||
# Prepare origin and origin_visit | # Prepare origin and origin_visit | ||||
origin = Origin(url=self.url) | origin = Origin(url=self.url) | ||||
try: | try: | ||||
self.storage.origin_add([origin]) | self.storage.origin_add([origin]) | ||||
visit = list( | visit = list( | ||||
self.storage.origin_visit_add( | self.storage.origin_visit_add( | ||||
[ | [ | ||||
OriginVisit( | OriginVisit( | ||||
Show All 37 Lines | def load(self) -> Dict: | ||||
status_load = "eventful" | status_load = "eventful" | ||||
except Exception as e: | except Exception as e: | ||||
self.storage.clear_buffers() | self.storage.clear_buffers() | ||||
load_exceptions.append(e) | load_exceptions.append(e) | ||||
sentry_sdk.capture_exception(e) | sentry_sdk.capture_exception(e) | ||||
logger.exception( | logger.exception( | ||||
"Failed loading branch %s for %s", branch_name, self.url | "Failed loading branch %s for %s", branch_name, self.url | ||||
) | ) | ||||
failed_branches.append(branch_name) | |||||
continue | continue | ||||
if revision_id is None: | if revision_id is None: | ||||
continue | continue | ||||
tmp_revisions[version].append((branch_name, revision_id)) | tmp_revisions[version].append((branch_name, revision_id)) | ||||
if load_exceptions: | if load_exceptions: | ||||
▲ Show 20 Lines • Show All 324 Lines • Show Last 20 Lines |
that's a weird way to put it imo. What about this?