Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/loader.py
| Show First 20 Lines • Show All 170 Lines • ▼ Show 20 Lines | def _store_origin_visit(self) -> None: | ||||
| """ | """ | ||||
| assert self.origin | assert self.origin | ||||
| self.storage.origin_add([self.origin]) | self.storage.origin_add([self.origin]) | ||||
| if not self.visit_date: # now as default visit_date if not provided | if not self.visit_date: # now as default visit_date if not provided | ||||
| self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) | self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) | ||||
| assert isinstance(self.visit_date, datetime.datetime) | assert isinstance(self.visit_date, datetime.datetime) | ||||
| assert isinstance(self.visit_type, str) | assert isinstance(self.visit_type, str) | ||||
| self.visit = self.storage.origin_visit_add( | self.visit = list( | ||||
| self.storage.origin_visit_add( | |||||
| [ | [ | ||||
| OriginVisit( | OriginVisit( | ||||
| origin=self.origin.url, date=self.visit_date, type=self.visit_type, | origin=self.origin.url, | ||||
| date=self.visit_date, | |||||
| type=self.visit_type, | |||||
| ) | ) | ||||
| ] | ] | ||||
| ) | |||||
| )[0] | )[0] | ||||
| @abstractmethod | @abstractmethod | ||||
| def prepare(self, *args, **kwargs) -> None: | def prepare(self, *args, **kwargs) -> None: | ||||
| """Second step executed by the loader to prepare some state needed by | """Second step executed by the loader to prepare some state needed by | ||||
| the loader. | the loader. | ||||
| """ | """ | ||||
| ▲ Show 20 Lines • Show All 104 Lines • ▼ Show 20 Lines | def load(self, *args, **kwargs) -> Dict[str, str]: | ||||
| except Exception: | except Exception: | ||||
| msg = "Cleaning up dangling data failed! Continue loading." | msg = "Cleaning up dangling data failed! Continue loading." | ||||
| self.log.warning(msg) | self.log.warning(msg) | ||||
| self.prepare_origin_visit(*args, **kwargs) | self.prepare_origin_visit(*args, **kwargs) | ||||
| self._store_origin_visit() | self._store_origin_visit() | ||||
| assert self.origin | assert self.origin | ||||
| assert self.visit.visit | |||||
| try: | try: | ||||
| self.prepare(*args, **kwargs) | self.prepare(*args, **kwargs) | ||||
| while True: | while True: | ||||
| more_data_to_fetch = self.fetch_data() | more_data_to_fetch = self.fetch_data() | ||||
| self.store_data() | self.store_data() | ||||
| if not more_data_to_fetch: | if not more_data_to_fetch: | ||||
| ▲ Show 20 Lines • Show All 101 Lines • ▼ Show 20 Lines | def store_data(self) -> None: | ||||
| contents.append(obj) | contents.append(obj) | ||||
| elif isinstance(obj, SkippedContent): | elif isinstance(obj, SkippedContent): | ||||
| skipped_contents.append(obj) | skipped_contents.append(obj) | ||||
| else: | else: | ||||
| raise TypeError(f"Unexpected content type: {obj}") | raise TypeError(f"Unexpected content type: {obj}") | ||||
| self.storage.skipped_content_add(skipped_contents) | self.storage.skipped_content_add(skipped_contents) | ||||
| self.storage.content_add(contents) | self.storage.content_add(contents) | ||||
| if self.has_directories(): | if self.has_directories(): | ||||
| self.storage.directory_add(self.get_directories()) | self.storage.directory_add(list(self.get_directories())) | ||||
| if self.has_revisions(): | if self.has_revisions(): | ||||
| self.storage.revision_add(self.get_revisions()) | self.storage.revision_add(list(self.get_revisions())) | ||||
| if self.has_releases(): | if self.has_releases(): | ||||
| self.storage.release_add(self.get_releases()) | self.storage.release_add(list(self.get_releases())) | ||||
| snapshot = self.get_snapshot() | snapshot = self.get_snapshot() | ||||
| self.storage.snapshot_add([snapshot]) | self.storage.snapshot_add([snapshot]) | ||||
| self.flush() | self.flush() | ||||
| self.loaded_snapshot_id = snapshot.id | self.loaded_snapshot_id = snapshot.id | ||||