Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/loader.py
| Show First 20 Lines • Show All 326 Lines • ▼ Show 20 Lines | def load(self) -> Dict: | ||||
| information from step 0., generate a snapshot and load it into the | information from step 0., generate a snapshot and load it into the | ||||
| Software Heritage archive | Software Heritage archive | ||||
| """ | """ | ||||
| status_load = "uneventful" # either: eventful, uneventful, failed | status_load = "uneventful" # either: eventful, uneventful, failed | ||||
| status_visit = "full" # either: partial, full | status_visit = "full" # either: partial, full | ||||
| tmp_revisions = {} # type: Dict[str, List] | tmp_revisions = {} # type: Dict[str, List] | ||||
| snapshot = None | snapshot = None | ||||
| failed_branches: List[str] = [] | |||||
| def finalize_visit() -> Dict[str, Any]: | def finalize_visit() -> Dict[str, Any]: | ||||
| """Finalize the visit: | """Finalize the visit: | ||||
| - flush eventual unflushed data to storage | - flush eventual unflushed data to storage | ||||
| - update origin visit's status | - update origin visit's status | ||||
| - return the task's status | - return the task's status | ||||
| Show All 12 Lines | def load(self) -> Dict: | ||||
| snapshot=snapshot_id, | snapshot=snapshot_id, | ||||
| ) | ) | ||||
| self.storage.origin_visit_status_add([visit_status]) | self.storage.origin_visit_status_add([visit_status]) | ||||
| result: Dict[str, Any] = { | result: Dict[str, Any] = { | ||||
| "status": status_load, | "status": status_load, | ||||
| } | } | ||||
| if snapshot_id: | if snapshot_id: | ||||
| result["snapshot_id"] = hash_to_hex(snapshot_id) | result["snapshot_id"] = hash_to_hex(snapshot_id) | ||||
| if failed_branches: | |||||
| logger.warning( | |||||
| "Summary report:\n- failed branches: %s\n" | |||||
| + "- Number of failed branches: %s", | |||||
| failed_branches, | |||||
| len(failed_branches), | |||||
| ) | |||||
vlorentz: that's a weird way to put it imo. What about this?
```
logger.warning(
"%d failed branches… | |||||
ardumontAuthorUnsubmitted Done Inline Actionsyeah, i'm biased. ardumont: yeah, i'm biased.
I did it so one does not have to scroll too much backwards (if the list of… | |||||
| return result | return result | ||||
| # Prepare origin and origin_visit | # Prepare origin and origin_visit | ||||
| origin = Origin(url=self.url) | origin = Origin(url=self.url) | ||||
| try: | try: | ||||
| self.storage.origin_add([origin]) | self.storage.origin_add([origin]) | ||||
| visit = list( | visit = list( | ||||
| self.storage.origin_visit_add( | self.storage.origin_visit_add( | ||||
| Show All 39 Lines | def load(self) -> Dict: | ||||
| status_load = "eventful" | status_load = "eventful" | ||||
| except Exception as e: | except Exception as e: | ||||
| self.storage.clear_buffers() | self.storage.clear_buffers() | ||||
| load_exceptions.append(e) | load_exceptions.append(e) | ||||
| sentry_sdk.capture_exception(e) | sentry_sdk.capture_exception(e) | ||||
| logger.exception( | logger.exception( | ||||
| "Failed loading branch %s for %s", branch_name, self.url | "Failed loading branch %s for %s", branch_name, self.url | ||||
| ) | ) | ||||
| failed_branches.append(branch_name) | |||||
| continue | continue | ||||
| if revision_id is None: | if revision_id is None: | ||||
| continue | continue | ||||
| tmp_revisions[version].append((branch_name, revision_id)) | tmp_revisions[version].append((branch_name, revision_id)) | ||||
| if load_exceptions: | if load_exceptions: | ||||
| ▲ Show 20 Lines • Show All 324 Lines • Show Last 20 Lines | |||||
that's a weird way to put it imo. What about this?
logger.warning( "%d failed branches: %s", len(failed_branches), failed_branches, )