Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/base/dowload.py
- This file was added.
| # Copyright (C) 2019 The Software Heritage developers | |||||
anlambert: I am not a big fan of using inheritance in derived package loaders implementation (CRAN, GNU, .. | |||||
Not Done Inline Actions
Yes, thank you. I did not find the right way to explicit that. We must try and keep separated what's logically separated (artefacts retrieval, ingestion, etc...).
Yes, that's what i call a collaborator. ardumont: > I am not a big fan of using inheritance in derived package loaders implementation (CRAN, GNU… | |||||
| # See the AUTHORS file at the top-level directory of this distribution | |||||
| # License: GNU General Public License version 3, or any later version | |||||
| # See top-level LICENSE file for more information | |||||
| import requests | |||||
| from .abstractattribute import AbstractAttribute | |||||
| try: | |||||
| from _version import __version__ | |||||
| except ImportError: | |||||
| __version__ = 'devel' | |||||
| # This file contains methods to check and remove archived package version | |||||
| # and download the new package version. | |||||
| class If_Modified_Since: | |||||
| """Uses if_modified_then header to check for archived packages | |||||
| """ | |||||
| def __init__(self): | |||||
| self.session = requests.session() | |||||
| self.params = { | |||||
| 'headers': { | |||||
| 'User-Agent': 'Software Heritage Tar Loader (%s)' % ( | |||||
| __version__ | |||||
| ) | |||||
| } | |||||
| } | |||||
| def known_artifacts(self, last_snapshot): | |||||
| """ | |||||
| Retrieve the known release versions for the npm package | |||||
| (i.e. those already ingested into the archive). | |||||
| Args | |||||
| last_snapshot (dict): Last snapshot for the visit | |||||
| Returns: | |||||
| dict: Dict whose keys are Tuple[filename, sha1] and values | |||||
| are revision ids. | |||||
| """ | |||||
| if not last_snapshot or 'branches' not in last_snapshot: | |||||
| return {} | |||||
| # retrieve only revisions (e.g the alias we do not want here) | |||||
| revs = [rev['target'] | |||||
| for rev in last_snapshot['branches'].values() | |||||
| if rev and rev['target_type'] == 'revision'] | |||||
| known_revisions = self.storage.revision_get(revs) | |||||
| ret = {} | |||||
| for revision in known_revisions: | |||||
| if not revision: # revision_get can return None | |||||
| continue | |||||
| if 'original_artifact' in revision['metadata']: # Fix me | |||||
| artifact = revision['metadata']['original_artifact'] | |||||
| ret[artifact['url']] = [revision['id'], | |||||
| artifact['time_last_visit']] | |||||
| return ret | |||||
| def filter_package_versions(self, tarballs, known_versions): | |||||
| """ | |||||
| Return the available versions for the focused package. | |||||
| Args: | |||||
| tarballs (list): may be provided by the loader, it enables | |||||
| to filter out versions already ingested in the archive. | |||||
| Returns: | |||||
| dict: A dict whose keys are Tuple[version, tarball_sha1] and | |||||
| values dicts with the following entries: | |||||
| * **name**: the package name | |||||
| * **version**: the package version | |||||
| * **filename**: the package source tarball filename | |||||
| * **sha1**: the package source tarball sha1 checksum | |||||
| * **date**: the package release date | |||||
| * **url**: the package source tarball download url | |||||
| """ | |||||
| # Done version is artifact | |||||
| versions = [] | |||||
| for release in tarballs: | |||||
| if release['url'] in known_versions: | |||||
| tarball_url = release['url'] | |||||
| tarball_request = self._request( | |||||
| tarball_url, | |||||
| time_last_visit=known_versions[tarball_url][1], | |||||
| throw_error=False) | |||||
| if tarball_request.status_code == 304: | |||||
| continue | |||||
| elif tarball_request.status_code == 404: | |||||
| self.log.debug('Tarball url %s returns a 404 error.', | |||||
| tarball_url) | |||||
| continue | |||||
| release['response'] = tarball_request | |||||
| versions.append(release) | |||||
| return versions | |||||
| def _request(self, url, time_last_visit, throw_error=True): | |||||
| """Request the remote tarball url. | |||||
| Args: | |||||
| url (str): Url (file or http*) | |||||
| Raises: | |||||
| ValueError in case of failing to query | |||||
| Returns: | |||||
| server response | |||||
| """ | |||||
| self.params['headers']['If-Modified-Since'] = time_last_visit | |||||
| response = self.session.get(url, **self.params, stream=True) | |||||
| if response.status_code != 200: | |||||
| raise ValueError("Fail to query '%s'. Reason: %s" % ( | |||||
| url, response.status_code)) | |||||
| return response | |||||
| def prepare_package_versions(self, known_versions=None): | |||||
| """ | |||||
| Instantiate a generator that will process a specific package released | |||||
| version at each iteration step. The following operations will be | |||||
| performed: | |||||
| 1. Create a temporary directory to download and extract the | |||||
| release tarball | |||||
| 2. Download the tarball | |||||
| 3. Check downloaded tarball integrity | |||||
| 4. Uncompress the tarball | |||||
| 5. Parse ``package.json`` file associated to the package version | |||||
| 6. Extract author from the parsed ``package.json`` file | |||||
| Args: | |||||
| known_versions (dict): may be provided by the loader, it enables | |||||
| to filter out versions already ingested in the archive. | |||||
| Yields: | |||||
| Tuple[dict, dict, dict, str]: tuples containing the following | |||||
| members: | |||||
| * a dict holding the parsed ``package.json`` file | |||||
| * a dict holding package author information | |||||
| * a dict holding package tarball information | |||||
| * a string holding the path of the uncompressed package to | |||||
| load into the archive | |||||
| """ | |||||
| new_versions = self.filter_package_versions(known_versions) | |||||
| for package_source_data in new_versions: | |||||
| tarball_request = package_source_data['response'] | |||||
| # To make things simple while creating revisions | |||||
| del package_source_data['response'] | |||||
| yield self._prepare_package_version(package_source_data, | |||||
| tarball_request) | |||||
| class compare_field: | |||||
Done Inline ActionsCan you suggest a better name for this class? nahimilega: Can you suggest a better name for this class? | |||||
| """Uses a field present in the metadata to check for archived packages. | |||||
| """ | |||||
| compare_field = AbstractAttribute("Field used to identify if the package" | |||||
| "version is previously archived") | |||||
| # eg for pypi loader compare_field = 'sha' | |||||
| def __init__(self): | |||||
| self.session = requests.session() | |||||
| self.params = { | |||||
| 'headers': { | |||||
| 'User-Agent': 'Software Heritage Tar Loader (%s)' % ( | |||||
| __version__ | |||||
| ) | |||||
| } | |||||
| } | |||||
| def _request(self, url, throw_error=True): | |||||
| """Request the remote tarball url. | |||||
| Args: | |||||
| url (str): Url (file or http*) | |||||
| Raises: | |||||
| ValueError in case of failing to query | |||||
| Returns: | |||||
| Tuple of local (filepath, hashes of filepath) | |||||
| """ | |||||
| response = self.session.get(url, **self.params, stream=True) | |||||
| if response.status_code != 200 and throw_error: | |||||
| raise ValueError("Fail to query '%s'. Reason: %s" % ( | |||||
| url, response.status_code)) | |||||
| return response | |||||
| def known_artifacts(self, last_snapshot): | |||||
| """ | |||||
Done Inline ActionsUse camel case like everywhere else. ardumont: Use camel case like everywhere else. | |||||
| Retrieve the known release versions for the npm package | |||||
| (i.e. those already ingested into the archive). | |||||
| Args | |||||
| last_snapshot (dict): Last snapshot for the visit | |||||
| Returns: | |||||
| dict: Dict whose keys are Tuple[filename, sha1] and values | |||||
| are revision ids. | |||||
| """ | |||||
| if not last_snapshot or 'branches' not in last_snapshot: | |||||
| return {} | |||||
Done Inline ActionsCan you suggest a better name for this class? nahimilega: Can you suggest a better name for this class?
| |||||
| # retrieve only revisions (e.g the alias we do not want here) | |||||
| revs = [rev['target'] | |||||
| for rev in last_snapshot['branches'].values() | |||||
Done Inline ActionsAs mentioned, remove that abstract attribute and just add a plain docstring with triple backquotes. ardumont: As mentioned, remove that abstract attribute and just add a plain docstring with triple… | |||||
| if rev and rev['target_type'] == 'revision'] | |||||
| known_revisions = self.storage.revision_get(revs) | |||||
| ret = {} | |||||
Done Inline ActionsThe docstring is below the variable definition. compare_field = None """Field used to identify if the p...""" ardumont: The docstring is below the variable definition.
```
compare_field = None
"""Field used to… | |||||
Not Done Inline Actionsping ;) ardumont: ping ;) | |||||
| for revision in known_revisions: | |||||
| if not revision: # revision_get can return None | |||||
| continue | |||||
| if 'original_artifact' in revision['metadata']: # Fix me | |||||
| artifact = revision['metadata']['original_artifact'] | |||||
| ret[artifact[self.compare_field]] = revision['id'] | |||||
| return ret | |||||
| def filter_package_versions(self, tarballs, known_versions): | |||||
| """ | |||||
| Return the available versions for the focused package. | |||||
| Args: | |||||
| tarballs (list): may be provided by the loader, it enables | |||||
| to filter out versions already ingested in the archive. | |||||
| Returns: | |||||
| dict: A dict whose keys are Tuple[version, tarball_sha1] and | |||||
| values dicts with the following entries: | |||||
| * **name**: the package name | |||||
| * **version**: the package version | |||||
| * **filename**: the package source tarball filename | |||||
| * **sha1**: the package source tarball sha1 checksum | |||||
| * **date**: the package release date | |||||
| * **url**: the package source tarball download url | |||||
| """ | |||||
| # Done version is artifact | |||||
| versions = [] | |||||
| for release in tarballs: | |||||
| if release[self.compare_field] in known_versions: | |||||
| continue | |||||
| versions.append(release) | |||||
| return versions | |||||
| def prepare_package_versions(self, tarballs, known_versions=None): | |||||
| """ | |||||
| Instantiate a generator that will process a specific package released | |||||
| version at each iteration step. The following operations will be | |||||
| performed: | |||||
| 1. Create a temporary directory to download and extract the | |||||
| release tarball | |||||
| 2. Download the tarball | |||||
| 3. Check downloaded tarball integrity | |||||
| 4. Uncompress the tarball | |||||
| 5. Parse ``package.json`` file associated to the package version | |||||
| 6. Extract author from the parsed ``package.json`` file | |||||
| Args: | |||||
| known_versions (dict): may be provided by the loader, it enables | |||||
| to filter out versions already ingested in the archive. | |||||
| Yields: | |||||
| Tuple[dict, dict, dict, str]: tuples containing the following | |||||
| members: | |||||
| * a dict holding the parsed ``package.json`` file | |||||
| * a dict holding package author information | |||||
| * a dict holding package tarball information | |||||
| * a string holding the path of the uncompressed package to | |||||
| load into the archive | |||||
| """ | |||||
| new_versions = self.filter_package_versions(tarballs, known_versions) | |||||
| for package_source_data in new_versions: | |||||
| # filter out version with missing tarball, | |||||
| # package visit will be marked as partial at the end of | |||||
| # the loading process | |||||
| tarball_url = package_source_data['url'] | |||||
| tarball_request = self._request(tarball_url, | |||||
| throw_error=False) | |||||
| if tarball_request.status_code == 404: | |||||
| self.log.debug('Tarball url %s returns a 404 error.', | |||||
| tarball_url) | |||||
| continue | |||||
| yield self._prepare_package_version(package_source_data, | |||||
| tarball_request) | |||||
Done Inline ActionsChanged known_versions=None to known_versions={} because if known_versions is None then it would create a problem later in code nahimilega: Changed `known_versions=None` to `known_versions={}` because if known_versions is None then it… | |||||
I am not a big fan of using inheritance in derived package loaders implementation (CRAN, GNU, ...) to declare the package tarballs retrieval behavior.
I would rather use a single class, named PackageDownloader for instance, that will internally be used by the PackageLoader class through composition instead.
Derived classes implementing real packages loader (GNU, CRAN, ..) should only inherit from the PackageLoader class and reimplement some hook methods to override the default behavior.
I need to further analyze what is implemented in that file before suggesting improvements.