Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/loader.py
| # Copyright (C) 2015-2020 The Software Heritage developers | # Copyright (C) 2015-2021 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from abc import ABCMeta, abstractmethod | from abc import ABCMeta, abstractmethod | ||||
| import datetime | import datetime | ||||
| import hashlib | import hashlib | ||||
| import logging | import logging | ||||
| ▲ Show 20 Lines • Show All 317 Lines • ▼ Show 20 Lines | def load(self, *args, **kwargs) -> Dict[str, str]: | ||||
| type=self.visit_type, | type=self.visit_type, | ||||
| date=now(), | date=now(), | ||||
| status=self.visit_status(), | status=self.visit_status(), | ||||
| snapshot=self.loaded_snapshot_id, | snapshot=self.loaded_snapshot_id, | ||||
| ) | ) | ||||
| self.storage.origin_visit_status_add([visit_status]) | self.storage.origin_visit_status_add([visit_status]) | ||||
| self.post_load() | self.post_load() | ||||
| except Exception: | except Exception: | ||||
| status = "partial" if self.loaded_snapshot_id else "failed" | |||||
| self.log.exception( | self.log.exception( | ||||
| "Loading failure, updating to `partial` status", | "Loading failure, updating to `%s` status", | ||||
| status, | |||||
vlorentz: or `failed` | |||||
| extra={"swh_task_args": args, "swh_task_kwargs": kwargs,}, | extra={"swh_task_args": args, "swh_task_kwargs": kwargs,}, | ||||
| ) | ) | ||||
| visit_status = OriginVisitStatus( | visit_status = OriginVisitStatus( | ||||
| origin=self.origin.url, | origin=self.origin.url, | ||||
| visit=self.visit.visit, | visit=self.visit.visit, | ||||
| type=self.visit_type, | type=self.visit_type, | ||||
| date=now(), | date=now(), | ||||
| status="partial", | status=status, | ||||
| snapshot=self.loaded_snapshot_id, | snapshot=self.loaded_snapshot_id, | ||||
| ) | ) | ||||
| self.storage.origin_visit_status_add([visit_status]) | self.storage.origin_visit_status_add([visit_status]) | ||||
| self.post_load(success=False) | self.post_load(success=False) | ||||
| return {"status": "failed"} | return {"status": "failed"} | ||||
| finally: | finally: | ||||
| self.flush() | self.flush() | ||||
| self.cleanup() | self.cleanup() | ||||
| ▲ Show 20 Lines • Show All 85 Lines • Show Last 20 Lines | |||||
or failed