Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/loader.py
| Show First 20 Lines • Show All 274 Lines • ▼ Show 20 Lines | def load(self) -> Dict: | ||||
| return {'status': 'failed'} | return {'status': 'failed'} | ||||
| try: | try: | ||||
| last_snapshot = self.last_snapshot() | last_snapshot = self.last_snapshot() | ||||
| logger.debug('last snapshot: %s', last_snapshot) | logger.debug('last snapshot: %s', last_snapshot) | ||||
| known_artifacts = self.known_artifacts(last_snapshot) | known_artifacts = self.known_artifacts(last_snapshot) | ||||
| logger.debug('known artifacts: %s', known_artifacts) | logger.debug('known artifacts: %s', known_artifacts) | ||||
| # Retrieve the default release version (the "latest" one) | |||||
| default_version = self.get_default_version() | |||||
| logger.debug('default version: %s', default_version) | |||||
| for version in self.get_versions(): # for each | for version in self.get_versions(): # for each | ||||
| logger.debug('version: %s', version) | logger.debug('version: %s', version) | ||||
| tmp_revisions[version] = [] | tmp_revisions[version] = [] | ||||
| # `p_` stands for `package_` | # `p_` stands for `package_` | ||||
| for branch_name, p_info in self.get_package_info(version): | for branch_name, p_info in self.get_package_info(version): | ||||
| logger.debug('package_info: %s', p_info) | logger.debug('package_info: %s', p_info) | ||||
| revision_id = self.resolve_revision_from( | revision_id = self.resolve_revision_from( | ||||
| known_artifacts, p_info['raw']) | known_artifacts, p_info['raw']) | ||||
| if revision_id is None: | if revision_id is None: | ||||
| (revision_id, loaded) = \ | (revision_id, loaded) = \ | ||||
| self._load_revision(p_info, origin) | self._load_revision(p_info, origin) | ||||
| if loaded: | if loaded: | ||||
| status_load = 'eventful' | status_load = 'eventful' | ||||
| else: | else: | ||||
| status_visit = 'partial' | status_visit = 'partial' | ||||
| if revision_id is None: | if revision_id is None: | ||||
| continue | continue | ||||
| tmp_revisions[version].append((branch_name, revision_id)) | tmp_revisions[version].append((branch_name, revision_id)) | ||||
| snapshot = self._load_snapshot(default_version, tmp_revisions) | |||||
| if hasattr(self.storage, 'flush'): | |||||
| self.storage.flush() | |||||
| except Exception: | except Exception: | ||||
| logger.exception('Fail to load %s' % self.url) | logger.exception('Fail to load %s' % self.url) | ||||
| status_visit = 'partial' | status_visit = 'partial' | ||||
| status_load = 'failed' | status_load = 'failed' | ||||
| finally: | finally: | ||||
lewo: I think we could be in trouble if this method raises an exception. So, maybe we could instead… | |||||
Done Inline ActionsLet's make the functional loader resilient to this and not the loader one. (i think try except within its own implementation). Same goes for default version by the way. ardumont: Let's make the functional loader resilient to this and not the loader one.
Since there is only… | |||||
| if tmp_revisions: | |||||
Done Inline Actions^ D2769 makes me wonder... ardumont: ^ D2769 makes me wonder... | |||||
Done Inline Actionsnvm that comment now, it's no longer relevant. ardumont: nvm that comment now, it's no longer relevant. | |||||
| snapshot = self._load_snapshot(tmp_revisions) | |||||
| if hasattr(self.storage, 'flush'): | |||||
| self.storage.flush() | |||||
| self.storage.origin_visit_update( | self.storage.origin_visit_update( | ||||
| origin=self.url, visit_id=visit.visit, status=status_visit, | origin=self.url, visit_id=visit.visit, status=status_visit, | ||||
| snapshot=snapshot and snapshot.id) | snapshot=snapshot and snapshot.id) | ||||
| result = { | result: Dict[str, Any] = { | ||||
| 'status': status_load, | 'status': status_load, | ||||
| } # type: Dict[str, Any] | } | ||||
| if snapshot: | if snapshot: | ||||
| result['snapshot_id'] = hash_to_hex(snapshot.id) | result['snapshot_id'] = hash_to_hex(snapshot.id) | ||||
| return result | return result | ||||
| def _load_revision(self, p_info, origin) -> Tuple[Optional[Sha1Git], bool]: | def _load_revision(self, p_info, origin) -> Tuple[Optional[Sha1Git], bool]: | ||||
| """Does all the loading of a revision itself: | """Does all the loading of a revision itself: | ||||
| * downloads a package and uncompresses it | * downloads a package and uncompresses it | ||||
| ▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | def _load_revision(self, p_info, origin) -> Tuple[Optional[Sha1Git], bool]: | ||||
| logger.debug('Revision: %s', revision) | logger.debug('Revision: %s', revision) | ||||
| self.storage.revision_add([revision]) | self.storage.revision_add([revision]) | ||||
| return (revision.id, True) | return (revision.id, True) | ||||
| def _load_snapshot( | def _load_snapshot( | ||||
| self, default_version: str, | self, revisions: Dict[str, List[Tuple[str, bytes]]]) -> Snapshot: | ||||
| revisions: Dict[str, List[Tuple[str, bytes]]]) -> Snapshot: | |||||
| """Build snapshot out of the current revisions stored. Then load it in | """Build snapshot out of the current revisions stored. Then load it in | ||||
| the storage. | the storage. | ||||
Done Inline ActionsIt'd be great the types (revisions. extra_branches) were a tad unified here. ardumont: It'd be great the types (revisions. extra_branches) were a tad unified here.
But let's keep… | |||||
| """ | """ | ||||
| # Retrieve the default release version (the "latest" one) | |||||
Not Done Inline ActionsIsn't there some edge cases where generating an empty snapshot is the right thing to do? olasd: Isn't there some edge cases where generating an empty snapshot is the right thing to do? | |||||
Done Inline Actionsno idea. That'd simplify but i don't know what's better. ardumont: no idea.
That'd simplify but i don't know what's better. | |||||
| default_version = self.get_default_version() | |||||
| logger.debug('default version: %s', default_version) | |||||
| logger.debug('revisions: %s', revisions) | logger.debug('revisions: %s', revisions) | ||||
| # Build and load the snapshot | # Build and load the snapshot | ||||
| branches = {} # type: Dict[bytes, Mapping[str, Any]] | branches = {} # type: Dict[bytes, Mapping[str, Any]] | ||||
| for version, branch_name_revisions in revisions.items(): | for version, branch_name_revisions in revisions.items(): | ||||
| if version == default_version and \ | if version == default_version and \ | ||||
| len(branch_name_revisions) == 1: | len(branch_name_revisions) == 1: | ||||
| # only 1 branch (no ambiguity), we can create an alias | # only 1 branch (no ambiguity), we can create an alias | ||||
| # branch 'HEAD' | # branch 'HEAD' | ||||
| Show All 31 Lines | |||||
I think we could be in trouble if this method raises an exception. So, maybe we could instead get extra_branches at the beginning of the load method, before loading any revisions: we are then sure to have extra_branches and we are able to catch exceptions of this method.