Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/pypi/loader.py
| Show All 30 Lines | ADDITIONAL_CONFIG = { | ||||
| 'temp_directory': ('str', '/tmp/swh.loader.pypi/'), | 'temp_directory': ('str', '/tmp/swh.loader.pypi/'), | ||||
| 'cache': ('bool', False), | 'cache': ('bool', False), | ||||
| 'cache_dir': ('str', ''), | 'cache_dir': ('str', ''), | ||||
| 'debug': ('bool', False), # NOT FOR PRODUCTION | 'debug': ('bool', False), # NOT FOR PRODUCTION | ||||
| } | } | ||||
| def __init__(self, client=None): | def __init__(self, client=None): | ||||
| super().__init__(logging_class='swh.loader.pypi.PyPILoader') | super().__init__(logging_class='swh.loader.pypi.PyPILoader') | ||||
| self.origin_id = None | |||||
| if not client: | if not client: | ||||
| temp_directory = self.config['temp_directory'] | temp_directory = self.config['temp_directory'] | ||||
| os.makedirs(temp_directory, exist_ok=True) | os.makedirs(temp_directory, exist_ok=True) | ||||
| self.temp_directory = mkdtemp( | self.temp_directory = mkdtemp( | ||||
| suffix='-%s' % os.getpid(), | suffix='-%s' % os.getpid(), | ||||
| prefix=TEMPORARY_DIR_PREFIX_PATTERN, | prefix=TEMPORARY_DIR_PREFIX_PATTERN, | ||||
| dir=temp_directory) | dir=temp_directory) | ||||
| self.pypi_client = PyPIClient( | self.pypi_client = PyPIClient( | ||||
| ▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | def prepare_origin_visit(self, project_name, project_url, | ||||
| """ | """ | ||||
| self.origin = { | self.origin = { | ||||
| 'url': project_url, | 'url': project_url, | ||||
| 'type': 'pypi', | 'type': 'pypi', | ||||
| } | } | ||||
| self.visit_date = None # loader core will populate it | self.visit_date = None # loader core will populate it | ||||
| def _known_artifacts(self, last_snapshot): | def _known_artifacts(self, last_snapshot): | ||||
| """Retrieve the known releases/artifact for the origin_id. | """Retrieve the known releases/artifact for the origin. | ||||
| Args | Args | ||||
| snapshot (dict): Last snapshot for the visit | snapshot (dict): Last snapshot for the visit | ||||
| Returns: | Returns: | ||||
| list of (filename, sha256) tuples. | list of (filename, sha256) tuples. | ||||
| """ | """ | ||||
| Show All 13 Lines | def _known_artifacts(self, last_snapshot): | ||||
| artifact = revision['metadata']['original_artifact'] | artifact = revision['metadata']['original_artifact'] | ||||
| ret[artifact['filename'], artifact['sha256']] = revision['id'] | ret[artifact['filename'], artifact['sha256']] = revision['id'] | ||||
| return ret | return ret | ||||
| def _last_snapshot(self): | def _last_snapshot(self): | ||||
| """Retrieve the last snapshot | """Retrieve the last snapshot | ||||
| """ | """ | ||||
| snapshot = self.storage.snapshot_get_latest(self.origin_id) | visit = self.storage.origin_visit_get_latest( | ||||
| if snapshot and snapshot.pop('next_branch', None): | self.origin, require_snapshot=True) | ||||
| snapshot = snapshot_get_all_branches(self.storage, snapshot['id']) | if visit: | ||||
| return snapshot | return snapshot_get_all_branches(self.storage, visit['snapshot']) | ||||
| def prepare(self, project_name, project_url, | def prepare(self, project_name, project_url, | ||||
| project_metadata_url=None): | project_metadata_url=None): | ||||
| """Keep reference to the origin url (project) and the | """Keep reference to the origin url (project) and the | ||||
| project metadata url | project metadata url | ||||
| Args: | Args: | ||||
| project_name (str): Project's simple name | project_name (str): Project's simple name | ||||
| ▲ Show 20 Lines • Show All 174 Lines • Show Last 20 Lines | |||||