diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py index 3e0746c..aea5c39 100644 --- a/swh/loader/package/loader.py +++ b/swh/loader/package/loader.py @@ -1,238 +1,254 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import datetime import tempfile import os from typing import Generator, Dict, Tuple, Sequence from swh.core.tarball import uncompress from swh.core.config import SWHConfig from swh.model.from_disk import Directory from swh.model.identifiers import ( revision_identifier, snapshot_identifier, identifier_to_bytes ) from swh.storage import get_storage +from swh.loader.core.converters import content_for_storage class PackageLoader: # Origin visit type (str) set by the loader visit_type = None def __init__(self, url): """Loader's constructor. This raises exception if the minimal required configuration is missing (cf. fn:`check` method). Args: url (str): Origin url to load data from """ self.config = SWHConfig.parse_config_file() self._check_configuration() self.storage = get_storage(**self.config['storage']) # FIXME: No more configuration documentation # Implicitily, this uses the SWH_CONFIG_FILENAME environment variable # loading mechanism - - self.origin = {'url': url} + self.url = url def _check_configuration(self): """Checks the minimal configuration required is set for the loader. If some required configuration is missing, exception detailing the issue is raised. """ if 'storage' not in self.config: raise ValueError( 'Misconfiguration, at least the storage key should be set') def get_versions(self) -> Sequence[str]: """Return the list of all published package versions. Returns: Sequence of published versions """ return [] def get_artifacts(self, version: str) -> Generator[ Tuple[str, str, Dict], None, None]: """Given a release version of a package, retrieve the associated artifact information for such version. Args: version: Package version Returns: (artifact filename, artifact uri, raw artifact metadata) """ return [] def fetch_artifact_archive( self, artifact_archive_path: str, dest: str) -> str: """Fetch artifact archive to a temporary folder and returns its path. Args: artifact_archive_path: Path to artifact archive to uncompress dest: Directory to write the downloaded archive to Returns: the locally retrieved artifact path """ - pass + return '' - def build_revision(self) -> Dict: + def build_revision( + self, a_metadata: Dict, a_uncompressed_path: str) -> Dict: """Build the revision dict Returns: SWH data dict """ return {} + def get_default_release(self) -> str: + """Retrieve the latest release version + + Returns: + Latest version + + """ + return '' + def load(self): """Load for a specific origin the associated contents. for each package version of the origin 1. Fetch the files for one package version By default, this can be implemented as a simple HTTP request. Loaders with more specific requirements can override this, e.g.: the PyPI loader checks the integrity of the downloaded files; the Debian loader has to download and check several files for one package version. 2. Extract the downloaded files By default, this would be a universal archive/tarball extraction. Loaders for specific formats can override this method (for instance, the Debian loader uses dpkg-source -x). 3. Convert the extracted directory to a set of Software Heritage objects Using swh.model.from_disk. 4. Extract the metadata from the unpacked directories This would only be applicable for "smart" loaders like npm (parsing the package.json), PyPI (parsing the PKG-INFO file) or Debian (parsing debian/changelog and debian/control). On "minimal-metadata" sources such as the GNU archive, the lister should provide the minimal set of metadata needed to populate the revision/release objects (authors, dates) as an argument to the task. 5. Generate the revision/release objects for the given version. From the data generated at steps 3 and 4. end for each 6. Generate and load the snapshot for the visit Using the revisions/releases collected at step 5., and the branch information from step 0., generate a snapshot and load it into the Software Heritage archive """ status_load = 'uneventful' # either: eventful, uneventful, failed status_visit = 'partial' # either: partial, full tmp_revisions = {} - # Prepare origin and origin_visit (method?) - origin = self.storage.origin_add([self.origin])[0] - visit = self.storage.origin_visit_add( - origin=origin, type=self.visit_type)['visit'] + # Prepare origin and origin_visit + origin = {'url': self.url} + self.storage.origin_add([origin]) + visit_date = datetime.datetime.now(tz=datetime.timezone.utc) + visit_id = self.storage.origin_visit_add( + origin=self.url, + date=visit_date, + type=self.visit_type)['visit'] # Retrieve the default release (the "latest" one) default_release = self.get_default_release() # FIXME: Add load exceptions handling for version in self.get_versions(): # for each tmp_revisions[version] = [] # `a_` stands for `artifact_` for a_filename, a_uri, a_metadata in self.get_artifacts(version): with tempfile.TemporaryDirectory() as tmpdir: a_path, a_computed_metadata = self.fetch_artifact_archive( a_uri, dest=tmpdir) uncompressed_path = os.path.join(tmpdir, 'src') uncompress(a_path, dest=uncompressed_path) directory = Directory.from_disk( - path=uncompressed_path, data=True) + path=uncompressed_path.encode('utf-8'), data=True) # FIXME: Try not to load the full raw content in memory objects = directory.collect() contents = objects['content'].values() - self.storage.content_add(contents) + self.storage.content_add( + map(content_for_storage, contents)) status_load = 'eventful' directories = objects['directory'].values() + self.storage.directory_add(directories) # FIXME: This should be release. cf. D409 discussion - revision = self.build_revision(uncompressed_path) + revision = self.build_revision( + a_metadata, uncompressed_path) revision.update({ 'type': 'tar', 'synthetic': True, 'directory': directory.hash, }) revision['metadata'].update({ 'original_artifact': a_metadata, 'hashes_artifact': a_computed_metadata }) revision['id'] = identifier_to_bytes( revision_identifier(revision)) - self.storage.revision_add(revision) + self.storage.revision_add([revision]) - tmp_revisions[version].append[{ + tmp_revisions[version].append({ 'filename': a_filename, 'target': revision['id'], - }] + }) # Build and load the snapshot branches = {} for version, v_branches in tmp_revisions.items(): if len(v_branches) == 1: - branch_name = 'releases/%s' % version - if version == default_release['version']: + branch_name = ('releases/%s' % version).encode('utf-8') + if version == default_release: branches[b'HEAD'] = { 'target_type': 'alias', - 'target': branch_name.encode('utf-8'), + 'target': branch_name, } branches[branch_name] = { 'target_type': 'revision', 'target': v_branches[0]['target'], } else: for x in v_branches: - branch_name = 'releases/%s/%s' % ( - version, v_branches['filename']) + branch_name = ('releases/%s/%s' % ( + version, v_branches['filename'])).encode('utf-8') branches[branch_name] = { 'target_type': 'revision', 'target': x['target'], } snapshot = { 'branches': branches } snapshot['id'] = identifier_to_bytes( snapshot_identifier(snapshot)) self.storage.snapshot_add([snapshot]) # come so far, we actually reached a full visit status_visit = 'full' # Update the visit's state - self.origin_visit_update( - origin=self.origin, - visit_id=visit['visit'], - status=status_visit, + self.storage.origin_visit_update( + origin=self.url, visit_id=visit_id, status=status_visit, snapshot=snapshot) return {'status': status_load} diff --git a/swh/loader/package/pypi.py b/swh/loader/package/pypi.py index 768f5ff..2d3494d 100644 --- a/swh/loader/package/pypi.py +++ b/swh/loader/package/pypi.py @@ -1,272 +1,277 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os from typing import Generator, Dict, Tuple, Sequence from urllib.parse import urlparse from pkginfo import UnpackedSDist import iso8601 import requests from swh.model.identifiers import normalize_timestamp from swh.model.hashutil import MultiHash, HASH_BLOCK_SIZE from swh.loader.package.loader import PackageLoader try: from swh.loader.core._version import __version__ except ImportError: __version__ = 'devel' DEFAULT_PARAMS = { 'headers': { 'User-Agent': 'Software Heritage Loader (%s)' % ( __version__ ) } } class PyPIClient: """PyPI api client. This deals with fetching json metadata about pypi projects. Args: url (str): PyPI instance's url (e.g: https://pypi.org/project/requests) api: - https://pypi.org/pypi/requests/json - https://pypi.org/pypi/requests/1.0.0/json (release description) """ def __init__(self, url): self.version = __version__ _url = urlparse(url) project_name = _url.path.split('/')[-1] self.url = '%s://%s/pypi/%s' % (_url.scheme, _url.netloc, project_name) self._session = None @property def session(self): if not self._session: self._session = requests.session() return self._session def _get(self, url: str) -> Dict: """Get query to the url. Args: url (str): Url Raises: ValueError in case of failing to query Returns: Response as dict if ok """ response = requests.get(url, **DEFAULT_PARAMS) if response.status_code != 200: raise ValueError("Fail to query '%s'. Reason: %s" % ( url, response.status_code)) return response.json() def info_project(self) -> Dict: """Given a url, retrieve the raw json response Returns: Main project information as dict. """ return self._get('%s/json' % self.url) def info_release(self, release: str) -> Dict: """Given a release version, retrieve the raw information for such release Args: release: Release version Returns: Release information as dict """ return self._get('%s/%s/json' % (self.url, release)) def download(url: str, dest: str) -> Tuple[str, Dict]: """Download a remote tarball from url, uncompresses and computes swh hashes on it. Args: url: Artifact uri to fetch, uncompress and hash dest: Directory to write the archive to Raises: ValueError in case of any error when fetching/computing Returns: Tuple of local (filepath, hashes of filepath) """ response = requests.get(url, **DEFAULT_PARAMS, stream=True) if response.status_code != 200: raise ValueError("Fail to query '%s'. Reason: %s" % ( url, response.status_code)) length = int(response.headers['content-length']) filepath = os.path.join(dest, os.path.basename(url)) h = MultiHash(length=length) with open(filepath, 'wb') as f: for chunk in response.iter_content(chunk_size=HASH_BLOCK_SIZE): h.update(chunk) f.write(chunk) actual_length = os.path.getsize(filepath) if length != actual_length: raise ValueError('Error when checking size: %s != %s' % ( length, actual_length)) # hashes = h.hexdigest() # actual_digest = hashes['sha256'] # if actual_digest != artifact['sha256']: # raise ValueError( # '%s %s: Checksum mismatched: %s != %s' % ( # project, version, artifact['sha256'], actual_digest)) return filepath, { 'length': length, **h.hexdigest() } def sdist_parse(dir_path: str) -> Dict: """Given an uncompressed path holding the pkginfo file, returns a pkginfo parsed structure as a dict. The release artifact contains at their root one folder. For example: $ tar tvf zprint-0.0.6.tar.gz drwxr-xr-x root/root 0 2018-08-22 11:01 zprint-0.0.6/ ... Args: dir_path (str): Path to the uncompressed directory representing a release artifact from pypi. Returns: the pkginfo parsed structure as a dict if any or None if none was present. """ # Retrieve the root folder of the archive if not os.path.exists(dir_path): return None lst = os.listdir(dir_path) if len(lst) == 0: return None project_dirname = lst[0] pkginfo_path = os.path.join(dir_path, project_dirname, 'PKG-INFO') if not os.path.exists(pkginfo_path): return None pkginfo = UnpackedSDist(pkginfo_path) raw = pkginfo.__dict__ raw.pop('filename') # this gets added with the ondisk location return raw def author(data: Dict) -> Dict: """Given a dict of project/release artifact information (coming from PyPI), returns an author subset. Args: data (dict): Representing either artifact information or release information. Returns: swh-model dict representing a person. """ name = data.get('author') email = data.get('author_email') if email: fullname = '%s <%s>' % (name, email) else: fullname = name if not fullname: return {'fullname': b'', 'name': None, 'email': None} fullname = fullname.encode('utf-8') if name is not None: name = name.encode('utf-8') if email is not None: email = email.encode('utf-8') return {'fullname': fullname, 'name': name, 'email': email} class PyPILoader(PackageLoader): """Load pypi origin's artifact releases into swh archive. """ visit_type = 'pypi' def __init__(self, url): super().__init__(url=url) self.client = PyPIClient(url) self._info = None @property def info(self) -> Dict: """Return the project metadata information (fetched from pypi registry) """ if not self._info: self._info = self.client.info_project() # dict return self._info def get_versions(self) -> Sequence[str]: return self.info['releases'].keys() + def get_default_release(self) -> str: + return self.info['info']['version'] + def get_artifacts(self, version: str) -> Generator[ Tuple[str, str, Dict], None, None]: for meta in self.info['releases'][version]: yield meta['filename'], meta['url'], meta def fetch_artifact_archive( self, artifact_uri: str, dest: str) -> Tuple[str, Dict]: return download(artifact_uri, dest=dest) - def build_revision(self, artifact_uncompressed_path: str) -> Dict: + def build_revision( + self, a_metadata: Dict, a_uncompressed_path: str) -> Dict: # Parse metadata (project, artifact metadata) - metadata = sdist_parse(artifact_uncompressed_path) - - # Build revision - name = metadata['version'].encode('utf-8') - message = metadata['message'].encode('utf-8') - message = b'%s: %s' % (name, message) if message else name + metadata = sdist_parse(a_uncompressed_path) + # from intrinsic metadata + name = metadata['version'] _author = author(metadata) - _date = normalize_timestamp( - int(iso8601.parse_date(metadata['date']).timestamp())) + + # from extrinsic metadata + message = a_metadata.get('comment_text', '') + message = '%s: %s' % (name, message) if message else name + date = normalize_timestamp( + int(iso8601.parse_date(a_metadata['upload_time']).timestamp())) + return { - 'name': name, - 'message': message, + 'message': message.encode('utf-8'), 'author': _author, - 'date': _date, + 'date': date, 'committer': _author, - 'committer_date': _date, + 'committer_date': date, 'parents': [], 'metadata': { 'intrinsic_metadata': metadata, } } diff --git a/swh/loader/package/tests/resources/tarballs/0805nexter-1.1.0.zip b/swh/loader/package/tests/resources/files.pythonhosted.org/0805nexter-1.1.0.zip similarity index 100% rename from swh/loader/package/tests/resources/tarballs/0805nexter-1.1.0.zip rename to swh/loader/package/tests/resources/files.pythonhosted.org/0805nexter-1.1.0.zip diff --git a/swh/loader/package/tests/resources/files.pythonhosted.org/0805nexter-1.2.0.zip b/swh/loader/package/tests/resources/files.pythonhosted.org/0805nexter-1.2.0.zip new file mode 100644 index 0000000..8638d33 Binary files /dev/null and b/swh/loader/package/tests/resources/files.pythonhosted.org/0805nexter-1.2.0.zip differ diff --git a/swh/loader/package/tests/resources/files.pythonhosted.org/0805nexter-1.3.0.zip b/swh/loader/package/tests/resources/files.pythonhosted.org/0805nexter-1.3.0.zip new file mode 100644 index 0000000..3fa6c3a Binary files /dev/null and b/swh/loader/package/tests/resources/files.pythonhosted.org/0805nexter-1.3.0.zip differ diff --git a/swh/loader/package/tests/resources/files.pythonhosted.org/0805nexter-1.4.0.zip b/swh/loader/package/tests/resources/files.pythonhosted.org/0805nexter-1.4.0.zip new file mode 100644 index 0000000..316ced2 Binary files /dev/null and b/swh/loader/package/tests/resources/files.pythonhosted.org/0805nexter-1.4.0.zip differ diff --git a/swh/loader/package/tests/resources/files.pythonhosted.org/packages_c4_a0_4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4_0805nexter-1.2.0.zip b/swh/loader/package/tests/resources/files.pythonhosted.org/packages_c4_a0_4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4_0805nexter-1.2.0.zip new file mode 120000 index 0000000..58026f3 --- /dev/null +++ b/swh/loader/package/tests/resources/files.pythonhosted.org/packages_c4_a0_4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4_0805nexter-1.2.0.zip @@ -0,0 +1 @@ +0805nexter-1.2.0.zip \ No newline at end of file diff --git a/swh/loader/package/tests/resources/files.pythonhosted.org/packages_ec_65_c0116953c9a3f47de89e71964d6c7b0c783b01f29fa3390584dbf3046b4d_0805nexter-1.1.0.zip b/swh/loader/package/tests/resources/files.pythonhosted.org/packages_ec_65_c0116953c9a3f47de89e71964d6c7b0c783b01f29fa3390584dbf3046b4d_0805nexter-1.1.0.zip new file mode 120000 index 0000000..e4b08b9 --- /dev/null +++ b/swh/loader/package/tests/resources/files.pythonhosted.org/packages_ec_65_c0116953c9a3f47de89e71964d6c7b0c783b01f29fa3390584dbf3046b4d_0805nexter-1.1.0.zip @@ -0,0 +1 @@ +0805nexter-1.1.0.zip \ No newline at end of file diff --git a/swh/loader/package/tests/resources/json/0805nexter+new-made-up-release.json b/swh/loader/package/tests/resources/json/0805nexter+new-made-up-release.json new file mode 100644 index 0000000..fbeb488 --- /dev/null +++ b/swh/loader/package/tests/resources/json/0805nexter+new-made-up-release.json @@ -0,0 +1,114 @@ +{ + "info": { + "author": "hgtkpython", + "author_email": "2868989685@qq.com", + "bugtrack_url": null, + "classifiers": [], + "description": "UNKNOWN", + "description_content_type": null, + "docs_url": null, + "download_url": "UNKNOWN", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "http://www.hp.com", + "keywords": null, + "license": "UNKNOWN", + "maintainer": null, + "maintainer_email": null, + "name": "0805nexter", + "package_url": "https://pypi.org/project/0805nexter/", + "platform": "UNKNOWN", + "project_url": "https://pypi.org/project/0805nexter/", + "project_urls": { + "Download": "UNKNOWN", + "Homepage": "http://www.hp.com" + }, + "release_url": "https://pypi.org/project/0805nexter/1.3.0/", + "requires_dist": null, + "requires_python": null, + "summary": "a simple printer of nested lest", + "version": "1.3.0" + }, + "last_serial": 1931736, + "releases": { + "1.1.0": [ + { + "comment_text": "", + "digests": { + "md5": "07fc93fc12821c1405c3483db88154af", + "sha256": "52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035" + }, + "downloads": -1, + "filename": "0805nexter-1.1.0.zip", + "has_sig": false, + "md5_digest": "07fc93fc12821c1405c3483db88154af", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 862, + "upload_time": "2016-01-31T05:28:42", + "url": "https://files.pythonhosted.org/packages/ec/65/c0116953c9a3f47de89e71964d6c7b0c783b01f29fa3390584dbf3046b4d/0805nexter-1.1.0.zip" + } + ], + "1.2.0": [ + { + "comment_text": "", + "digests": { + "md5": "89123c78bd5d3f61cb8f46029492b18a", + "sha256": "49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709" + }, + "downloads": -1, + "filename": "0805nexter-1.2.0.zip", + "has_sig": false, + "md5_digest": "89123c78bd5d3f61cb8f46029492b18a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 898, + "upload_time": "2016-01-31T05:51:25", + "url": "https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip" + } + ], + "1.3.0": [ + { + "comment_text": "Made up release 1.3.0 for swh-loader-pypi purposes", + "digests": { + "md5": "54d9750a1ab7ab82cd8c460c2c6c0ecc", + "sha256": "7097c49fb8ec24a7aaab54c3dbfbb5a6ca1431419d9ee0f6c363d9ad01d2b8b1" + }, + "downloads": -1, + "filename": "0805nexter-1.3.0.zip", + "has_sig": false, + "md5_digest": "54d9750a1ab7ab82cd8c460c2c6c0ecc", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 1370, + "upload_time": "2018-09-17T16:18:01", + "url": "https://files.pythonhosted.org/packages/70/97/c49fb8ec24a7aaab54c3dbfbb5a6ca1431419d9ee0f6c363d9ad01d2b8b1/0805nexter-1.3.0.zip" + } + ] + }, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "89123c78bd5d3f61cb8f46029492b18a", + "sha256": "49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709" + }, + "downloads": -1, + "filename": "0805nexter-1.2.0.zip", + "has_sig": false, + "md5_digest": "89123c78bd5d3f61cb8f46029492b18a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 898, + "upload_time": "2016-01-31T05:51:25", + "url": "https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip" + } + ] +} diff --git a/swh/loader/package/tests/resources/json/0805nexter-unpublished-release.json b/swh/loader/package/tests/resources/json/0805nexter-unpublished-release.json new file mode 100644 index 0000000..c9d0197 --- /dev/null +++ b/swh/loader/package/tests/resources/json/0805nexter-unpublished-release.json @@ -0,0 +1,116 @@ +{ + "info": { + "author": "hgtkpython", + "author_email": "2868989685@qq.com", + "bugtrack_url": null, + "classifiers": [], + "description": "UNKNOWN", + "description_content_type": null, + "docs_url": null, + "download_url": "UNKNOWN", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "http://www.hp.com", + "keywords": null, + "license": "UNKNOWN", + "maintainer": null, + "maintainer_email": null, + "name": "0805nexter", + "package_url": "https://pypi.org/project/0805nexter/", + "platform": "UNKNOWN", + "project_url": "https://pypi.org/project/0805nexter/", + "project_urls": { + "Download": "UNKNOWN", + "Homepage": "http://www.hp.com" + }, + "release_url": "https://pypi.org/project/0805nexter/1.4.0/", + "requires_dist": null, + "requires_python": null, + "summary": "a simple printer of nested lest", + "version": "1.4.0" + }, + "last_serial": 1931736, + "releases": { + "1.1.0": [], + "1.2.0": [ + { + "comment_text": "", + "digests": { + "md5": "89123c78bd5d3f61cb8f46029492b18a", + "sha256": "49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709" + }, + "downloads": -1, + "filename": "0805nexter-1.2.0.zip", + "has_sig": false, + "md5_digest": "89123c78bd5d3f61cb8f46029492b18a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 898, + "upload_time": "2016-01-31T05:51:25", + "url": "https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip" + } + ], + "1.3.0": [ + { + "comment_text": "Made up release 1.3.0 for swh-loader-pypi purposes", + "digests": { + "md5": "54d9750a1ab7ab82cd8c460c2c6c0ecc", + "sha256": "7097c49fb8ec24a7aaab54c3dbfbb5a6ca1431419d9ee0f6c363d9ad01d2b8b1" + }, + "downloads": -1, + "filename": "0805nexter-1.3.0.zip", + "has_sig": false, + "md5_digest": "54d9750a1ab7ab82cd8c460c2c6c0ecc", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 1370, + "upload_time": "2018-09-17T16:18:01", + "url": "https://files.pythonhosted.org/packages/70/97/c49fb8ec24a7aaab54c3dbfbb5a6ca1431419d9ee0f6c363d9ad01d2b8b1/0805nexter-1.3.0.zip" + } + ], + "1.4.0": [ + { + "comment_text": "1.4.0: Made up release for swh-loader-pypi test purposes", + "digests": { + "md5": "a30afef2f605ea837e6d5820412d5a7b", + "sha256": "4f6e85d0f9f04c00429ddcb25530f5c125fa169bd470cda9f04daa40862eb712" + }, + "downloads": -1, + "filename": "0805nexter-1.4.0.zip", + "has_sig": false, + "md5_digest": "54d9750a1ab7ab82cd8c460c2c6c0ecc", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 1370, + "upload_time": "2018-09-18T09:18:01", + "url": "https://files.pythonhosted.org/packages/4f/6e/85d0f9f04c00429ddcb25530f5c125fa169bd470cda9f04daa40862eb712/0805nexter-1.4.0.zip" + } + ] + + }, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "89123c78bd5d3f61cb8f46029492b18a", + "sha256": "49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709" + }, + "downloads": -1, + "filename": "0805nexter-1.2.0.zip", + "has_sig": false, + "md5_digest": "89123c78bd5d3f61cb8f46029492b18a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 898, + "upload_time": "2016-01-31T05:51:25", + "url": "https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip" + } + ] +} diff --git a/swh/loader/package/tests/resources/pypi.org/pypi_0805nexter_json b/swh/loader/package/tests/resources/pypi.org/pypi_0805nexter_json new file mode 100644 index 0000000..357bf16 --- /dev/null +++ b/swh/loader/package/tests/resources/pypi.org/pypi_0805nexter_json @@ -0,0 +1,95 @@ +{ + "info": { + "author": "hgtkpython", + "author_email": "2868989685@qq.com", + "bugtrack_url": null, + "classifiers": [], + "description": "UNKNOWN", + "description_content_type": null, + "docs_url": null, + "download_url": "UNKNOWN", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "http://www.hp.com", + "keywords": null, + "license": "UNKNOWN", + "maintainer": null, + "maintainer_email": null, + "name": "0805nexter", + "package_url": "https://pypi.org/project/0805nexter/", + "platform": "UNKNOWN", + "project_url": "https://pypi.org/project/0805nexter/", + "project_urls": { + "Download": "UNKNOWN", + "Homepage": "http://www.hp.com" + }, + "release_url": "https://pypi.org/project/0805nexter/1.2.0/", + "requires_dist": null, + "requires_python": null, + "summary": "a simple printer of nested lest", + "version": "1.2.0" + }, + "last_serial": 1931736, + "releases": { + "1.1.0": [ + { + "comment_text": "", + "digests": { + "md5": "07fc93fc12821c1405c3483db88154af", + "sha256": "52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035" + }, + "downloads": -1, + "filename": "0805nexter-1.1.0.zip", + "has_sig": false, + "md5_digest": "07fc93fc12821c1405c3483db88154af", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 862, + "upload_time": "2016-01-31T05:28:42", + "url": "https://files.pythonhosted.org/packages/ec/65/c0116953c9a3f47de89e71964d6c7b0c783b01f29fa3390584dbf3046b4d/0805nexter-1.1.0.zip" + } + ], + "1.2.0": [ + { + "comment_text": "", + "digests": { + "md5": "89123c78bd5d3f61cb8f46029492b18a", + "sha256": "49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709" + }, + "downloads": -1, + "filename": "0805nexter-1.2.0.zip", + "has_sig": false, + "md5_digest": "89123c78bd5d3f61cb8f46029492b18a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 898, + "upload_time": "2016-01-31T05:51:25", + "url": "https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip" + } + ] + }, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "89123c78bd5d3f61cb8f46029492b18a", + "sha256": "49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709" + }, + "downloads": -1, + "filename": "0805nexter-1.2.0.zip", + "has_sig": false, + "md5_digest": "89123c78bd5d3f61cb8f46029492b18a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 898, + "upload_time": "2016-01-31T05:51:25", + "url": "https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip" + } + ] +} diff --git a/swh/loader/package/tests/test_pypi.py b/swh/loader/package/tests/test_pypi.py index 30ef814..ffca5c3 100644 --- a/swh/loader/package/tests/test_pypi.py +++ b/swh/loader/package/tests/test_pypi.py @@ -1,318 +1,358 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os - -import pytest +import re from os import path +from urllib.parse import urlparse + +import pytest from swh.core.tarball import uncompress from swh.loader.package.pypi import ( PyPILoader, PyPIClient, author, sdist_parse, download ) +DATADIR = path.join(path.abspath(path.dirname(__file__)), 'resources') + def test_author_basic(): data = { 'author': "i-am-groot", 'author_email': 'iam@groot.org', } actual_author = author(data) expected_author = { 'fullname': b'i-am-groot ', 'name': b'i-am-groot', 'email': b'iam@groot.org', } assert actual_author == expected_author def test_author_empty_email(): data = { 'author': 'i-am-groot', 'author_email': '', } actual_author = author(data) expected_author = { 'fullname': b'i-am-groot', 'name': b'i-am-groot', 'email': b'', } assert actual_author == expected_author def test_author_empty_name(): data = { 'author': "", 'author_email': 'iam@groot.org', } actual_author = author(data) expected_author = { 'fullname': b' ', 'name': b'', 'email': b'iam@groot.org', } assert actual_author == expected_author def test_author_malformed(): data = { 'author': "['pierre', 'paul', 'jacques']", 'author_email': None, } actual_author = author(data) expected_author = { 'fullname': b"['pierre', 'paul', 'jacques']", 'name': b"['pierre', 'paul', 'jacques']", 'email': None, } assert actual_author == expected_author def test_author_malformed_2(): data = { 'author': '[marie, jeanne]', 'author_email': '[marie@some, jeanne@thing]', } actual_author = author(data) expected_author = { 'fullname': b'[marie, jeanne] <[marie@some, jeanne@thing]>', 'name': b'[marie, jeanne]', 'email': b'[marie@some, jeanne@thing]', } assert actual_author == expected_author def test_author_malformed_3(): data = { 'author': '[marie, jeanne, pierre]', 'author_email': '[marie@somewhere.org, jeanne@somewhere.org]', } actual_author = author(data) expected_author = { 'fullname': b'[marie, jeanne, pierre] <[marie@somewhere.org, jeanne@somewhere.org]>', # noqa 'name': b'[marie, jeanne, pierre]', 'email': b'[marie@somewhere.org, jeanne@somewhere.org]', } actual_author == expected_author # configuration error # -def test_badly_configured_loader_raise(): +def test_badly_configured_loader_raise(monkeypatch): """Badly configured loader should raise""" - assert 'SWH_CONFIG_FILENAME' in os.environ # cf. tox.ini - del os.environ['SWH_CONFIG_FILENAME'] + monkeypatch.delenv('SWH_CONFIG_FILENAME') with pytest.raises(ValueError) as e: PyPILoader(url='some-url') assert 'Misconfiguration' in e.value.args[0] def test_pypiclient_init(): """Initialization should set the api's base project url""" project_url = 'https://pypi.org/project/requests' expected_base_url = 'https://pypi.org/pypi/requests' pypi_client = PyPIClient(url=project_url) assert pypi_client.url == expected_base_url def test_pypiclient_failure(requests_mock): """Failure to fetch info/release information should raise""" project_url = 'https://pypi.org/project/requests' pypi_client = PyPIClient(url=project_url) expected_status_code = 400 info_url = '%s/json' % pypi_client.url requests_mock.get(info_url, status_code=expected_status_code) with pytest.raises(ValueError) as e0: pypi_client.info_project() assert e0.value.args[0] == "Fail to query '%s'. Reason: %s" % ( info_url, expected_status_code ) expected_status_code = 404 release_url = '%s/3.0.0/json' % pypi_client.url requests_mock.get(release_url, status_code=expected_status_code) with pytest.raises(ValueError) as e1: pypi_client.info_release("3.0.0") assert e1.value.args[0] == "Fail to query '%s'. Reason: %s" % ( release_url, expected_status_code ) def test_pypiclient(requests_mock): """Fetching info/release info should be ok""" pypi_client = PyPIClient('https://pypi.org/project/requests') info_url = '%s/json' % pypi_client.url requests_mock.get(info_url, text='{"version": "0.0.1"}') actual_info = pypi_client.info_project() assert actual_info == { 'version': '0.0.1', } release_url = '%s/2.0.0/json' % pypi_client.url requests_mock.get(release_url, text='{"version": "2.0.0"}') actual_release_info = pypi_client.info_release("2.0.0") assert actual_release_info == { 'version': '2.0.0', } -resources = path.abspath(path.dirname(__file__)) -resource_json = path.join(resources, 'resources/json') -resource_archives = path.join(resources, 'resources/tarballs') - - @pytest.mark.fs def test_sdist_parse(tmp_path): """Parsing existing archive's PKG-INFO should yield results""" uncompressed_archive_path = str(tmp_path) - archive_path = path.join(resource_archives, '0805nexter-1.1.0.zip') + archive_path = path.join( + DATADIR, 'files.pythonhosted.org', '0805nexter-1.1.0.zip') uncompress(archive_path, dest=uncompressed_archive_path) actual_sdist = sdist_parse(uncompressed_archive_path) expected_sdist = { 'metadata_version': '1.0', 'name': '0805nexter', 'version': '1.1.0', 'summary': 'a simple printer of nested lest', 'home_page': 'http://www.hp.com', 'author': 'hgtkpython', 'author_email': '2868989685@qq.com', 'platforms': ['UNKNOWN'], } assert actual_sdist == expected_sdist @pytest.mark.fs def test_sdist_parse_failures(tmp_path): """Parsing inexistant path/archive/PKG-INFO yield None""" # inexistant first level path assert sdist_parse('/something-inexistant') is None # inexistant second level path (as expected by pypi archives) assert sdist_parse(tmp_path) is None # inexistant PKG-INFO within second level path existing_path_no_pkginfo = str(tmp_path / 'something') os.mkdir(existing_path_no_pkginfo) assert sdist_parse(tmp_path) is None @pytest.mark.fs def test_download_fail_to_download(tmp_path, requests_mock): url = 'https://pypi.org/pypi/arrow/json' status_code = 404 requests_mock.get(url, status_code=status_code) with pytest.raises(ValueError) as e: download(url, tmp_path) assert e.value.args[0] == "Fail to query '%s'. Reason: %s" % ( url, status_code) @pytest.mark.fs def test_download_fail_length_mismatch(tmp_path, requests_mock): """Mismatch length after download should raise """ filename = 'requests-0.0.1.tar.gz' url = 'https://pypi.org/pypi/requests/%s' % filename data = 'this is something' wrong_size = len(data) - 3 requests_mock.get(url, text=data, headers={ 'content-length': str(wrong_size) # wrong size! }) with pytest.raises(ValueError) as e: download(url, dest=str(tmp_path)) assert e.value.args[0] == "Error when checking size: %s != %s" % ( wrong_size, len(data) ) @pytest.mark.fs def test_download_ok(tmp_path, requests_mock): """Download without issue should provide filename and hashes""" filename = 'requests-0.0.1.tar.gz' url = 'https://pypi.org/pypi/requests/%s' % filename data = 'this is something' requests_mock.get(url, text=data, headers={ 'content-length': str(len(data)) }) actual_filepath, actual_hashes = download(url, dest=str(tmp_path)) actual_filename = os.path.basename(actual_filepath) assert actual_filename == filename assert actual_hashes['length'] == len(data) assert actual_hashes['sha1'] == 'fdd1ce606a904b08c816ba84f3125f2af44d92b2' assert (actual_hashes['sha256'] == '1d9224378d77925d612c9f926eb9fb92850e6551def8328011b6a972323298d5') @pytest.mark.fs def test_download_fail_hashes_mismatch(tmp_path, requests_mock): """Mismatch hash after download should raise """ pass # LOADER SCENARIO # + +def get_response_cb(request, context): + """""" + url = urlparse(request.url) + dirname = url.hostname # pypi.org | files.pythonhosted.org + # url.path: pypi//json -> local file: pypi__json + filename = url.path[1:].replace('/', '_') + filepath = path.join(DATADIR, dirname, filename) + fd = open(filepath, 'rb') + context.headers['content-length'] = str(os.path.getsize(filepath)) + return fd + # "edge" cases (for the same origin) # + +def test_no_release_artifact(requests_mock): + pass + + # no release artifact: # {visit full, status: uneventful, no contents, etc...} # problem during loading: # {visit: partial, status: uneventful, no snapshot} # problem during loading: failure early enough in between swh contents... # some contents (contents, directories, etc...) have been written in storage # {visit: partial, status: eventful, no snapshot} # problem during loading: failure late enough we can have snapshots (some # revisions are written in storage already) # {visit: partial, status: eventful, snapshot} # "normal" cases (for the same origin) # -# release artifact, no prior visit -# {visit full, status eventful, snapshot} +def test_release_artifact_no_prior_visit(requests_mock): + """With no prior visit, load a pypi project ends up with 1 snapshot + + """ + assert 'SWH_CONFIG_FILENAME' in os.environ # cf. tox.ini + + loader = PyPILoader('https://pypi.org/project/0805nexter') + requests_mock.get(re.compile('https://'), + body=get_response_cb) + + actual_load_status = loader.load() + + assert actual_load_status == {'status': 'eventful'} + + stats = loader.storage.stat_counters() + assert { + 'content': 6, + 'directory': 4, + 'origin': 1, + 'origin_visit': 1, + 'person': 1, + 'release': 0, + 'revision': 2, + 'skipped_content': 0, + 'snapshot': 1 + } == stats # release artifact, no new artifact # {visit full, status uneventful, same snapshot as before} # release artifact, new artifact # {visit full, status full, new snapshot with shared history as prior snapshot} # release artifact, old artifact with different checksums # {visit full, status full, new snapshot with shared history and some new # different history}