diff --git a/swh/loader/pypi/tests/resources/0805nexter.json b/swh/loader/pypi/tests/resources/0805nexter.json new file mode 100644 index 0000000..52c8520 --- /dev/null +++ b/swh/loader/pypi/tests/resources/0805nexter.json @@ -0,0 +1 @@ +{"info":{"author":"hgtkpython","author_email":"2868989685@qq.com","bugtrack_url":null,"classifiers":[],"description":"UNKNOWN","description_content_type":null,"docs_url":null,"download_url":"UNKNOWN","downloads":{"last_day":-1,"last_month":-1,"last_week":-1},"home_page":"http://www.hp.com","keywords":null,"license":"UNKNOWN","maintainer":null,"maintainer_email":null,"name":"0805nexter","package_url":"https://pypi.org/project/0805nexter/","platform":"UNKNOWN","project_url":"https://pypi.org/project/0805nexter/","project_urls":{"Download":"UNKNOWN","Homepage":"http://www.hp.com"},"release_url":"https://pypi.org/project/0805nexter/1.2.0/","requires_dist":null,"requires_python":null,"summary":"a simple printer of nested lest","version":"1.2.0"},"last_serial":1931736,"releases":{"1.1.0":[{"comment_text":"","digests":{"md5":"07fc93fc12821c1405c3483db88154af","sha256":"52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035"},"downloads":-1,"filename":"0805nexter-1.1.0.zip","has_sig":false,"md5_digest":"07fc93fc12821c1405c3483db88154af","packagetype":"sdist","python_version":"source","requires_python":null,"size":862,"upload_time":"2016-01-31T05:28:42","url":"https://files.pythonhosted.org/packages/ec/65/c0116953c9a3f47de89e71964d6c7b0c783b01f29fa3390584dbf3046b4d/0805nexter-1.1.0.zip"}],"1.2.0":[{"comment_text":"","digests":{"md5":"89123c78bd5d3f61cb8f46029492b18a","sha256":"49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709"},"downloads":-1,"filename":"0805nexter-1.2.0.zip","has_sig":false,"md5_digest":"89123c78bd5d3f61cb8f46029492b18a","packagetype":"sdist","python_version":"source","requires_python":null,"size":898,"upload_time":"2016-01-31T05:51:25","url":"https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip"}]},"urls":[{"comment_text":"","digests":{"md5":"89123c78bd5d3f61cb8f46029492b18a","sha256":"49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709"},"downloads":-1,"filename":"0805nexter-1.2.0.zip","has_sig":false,"md5_digest":"89123c78bd5d3f61cb8f46029492b18a","packagetype":"sdist","python_version":"source","requires_python":null,"size":898,"upload_time":"2016-01-31T05:51:25","url":"https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip"}]} \ No newline at end of file diff --git a/swh/loader/pypi/tests/test_model.py b/swh/loader/pypi/tests/test_model.py index 5ce5759..84fbd4d 100644 --- a/swh/loader/pypi/tests/test_model.py +++ b/swh/loader/pypi/tests/test_model.py @@ -1,143 +1,148 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import os import json +import tempfile +import shutil from unittest import TestCase from nose.tools import istest -from swh.loader.pypi.model import PyPiProject +from swh.loader.pypi.model import PyPiProject, author +from swh.loader.pypi.client import PyPiClient, _project_pkginfo -class MockPyPiClient: - def release(self, project, release): - """Simulate the computation of the release object. +class PyPiClientWithCache(PyPiClient): + """Force the use of the cache to bypass pypi calls - The production code will trigger a query to the pypi api. + """ + def __init__(self, temp_directory, cache_dir): + super().__init__(temp_directory=temp_directory, + cache=True, cache_dir=cache_dir) - """ - if release == '0.1': - return { - 'info': { - 'home_page': 'something', - 'description': 'awesome python package', - 'summary': 'awesome python package', - 'license': '', - 'package_url': '', - 'project_url': '', - 'project_urls': { - 'Homepage': '' - }, - }, - 'releases': { - }, - } - - def fetch_release(self, project, release): - """Avoid fetching and writing to disk, simply returns the release - object - - The production code will trigger the raw archive fetch and - writes to temporary disk, we avoid this here. - - """ - return release +RESOURCES_PATH = './swh/loader/pypi/tests/resources' class ModelTest(TestCase): def setUp(self): - with open('./swh/loader/pypi/tests/test_model_data.json') as f: - self.data = json.load(f) - + project = '0805nexter' + project_metadata_file = '%s/%s.json' % (RESOURCES_PATH, project) + + with open(project_metadata_file) as f: + data = json.load(f) + + release_files = [] + for f in os.listdir(RESOURCES_PATH): + if f.endswith('.json'): + continue + file = os.path.join(RESOURCES_PATH, f) + release_files.append(file) + + temp_dir = tempfile.mkdtemp( + dir='/tmp/', prefix='swh.loader.pypi.tests-') + project_metadata_url = 'https://pypi.org/pypi/%s/json' % project + # Will use the pypi with cache + client = PyPiClientWithCache( + temp_directory=temp_dir, cache_dir=RESOURCES_PATH) self.project = PyPiProject( - client=MockPyPiClient(), - project='7xydothis', - project_metadata_url='https://pypi.org/pypi/7xydothis/json', - data=self.data) + client=client, + project=project, + project_metadata_url=project_metadata_url, + data=data) + + self.data = data + self.temp_dir = temp_dir + self.project_name = project + + def tearDown(self): + if os.path.exists(self.temp_dir): + shutil.rmtree(self.temp_dir) @istest def info(self): actual_info = self.project.info() expected_info = { 'home_page': self.data['info']['home_page'], 'description': self.data['info']['description'], 'summary': self.data['info']['summary'], 'license': self.data['info']['license'], 'package_url': self.data['info']['package_url'], 'project_url': self.data['info']['project_url'], 'upstream': self.data['info']['project_urls']['Homepage'], } self.assertEqual(expected_info, actual_info) @istest def author(self): - actual_author = self.project.author() + info = self.data['info'] + actual_author = author(info) - name = self.data['info']['author'].encode('utf-8') - email = self.data['info']['author_email'].encode('utf-8') + name = info['author'].encode('utf-8') + email = info['author_email'].encode('utf-8') expected_author = { 'fullname': b'%s <%s>' % (name, email), 'name': name, 'email': email, } self.assertEqual(expected_author, actual_author) @istest def releases(self): actual_releases = self.project.releases() - release0 = self.data['releases']['0.1'][0] - release1 = self.data['releases']['0.1.1'][0] - self.maxDiff = None - release_011_info = self.project.info() - expected_releases = [ - ('0.1', { - 'info': { - 'home_page': 'something', - 'description': 'awesome python package', - 'summary': 'awesome python package', - 'license': '', - 'package_url': '', - 'project_url': '', - 'upstream': '', - }, - 'release': { - 'name': '0.1', - 'message': release0['comment_text'], - 'sha256': release0['digests']['sha256'], - 'size': release0['size'], - 'filename': release0['filename'], - 'url': release0['url'], - 'date': release0['upload_time'], - } - }), - ('0.1.1', { - 'info': release_011_info, - 'release': { - 'name': '0.1.1', - 'message': release1['comment_text'], - 'sha256': release1['digests']['sha256'], - 'size': release1['size'], - 'filename': release1['filename'], - 'url': release1['url'], - 'date': release1['upload_time'], - } - }) - ] - - self.assertEqual(expected_releases, list(actual_releases)) - - @istest - def releases_unexpected_release_format(self): - data = self.data.copy() - data['releases']['0.1'].append({'anything': 'really to break'}) + expected_releases = { + '1.1.0': { + 'archive_type': 'zip', + 'blake2s256': 'df9413bde66e6133b10cadefad6fcf9cbbc369b47831089112c846d79f14985a', # noqa + 'date': '2016-01-31T05:28:42', + 'filename': '0805nexter-1.1.0.zip', + 'message': '', + 'name': '1.1.0', + 'sha1': '127d8697db916ba1c67084052196a83319a25000', + 'sha1_git': '4b8f1350e6d9fa00256e974ae24c09543d85b196', + 'sha256': '52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035', # noqa + 'size': 862, + 'url': 'https://files.pythonhosted.org/packages/ec/65/c0116953c9a3f47de89e71964d6c7b0c783b01f29fa3390584dbf3046b4d/0805nexter-1.1.0.zip', # noqa + }, + '1.2.0': { + 'archive_type': 'zip', + 'blake2s256': '67010586b5b9a4aaa3b1c386f9dc8b4c99e6e40f37732a717a5f9b9b1185e588', # noqa + 'date': '2016-01-31T05:51:25', + 'filename': '0805nexter-1.2.0.zip', + 'message': '', + 'name': '1.2.0', + 'sha1': 'd55238554b94da7c5bf4a349ece0fe3b2b19f79c', + 'sha1_git': '8638d33a96cb25d8319af21417f00045ec6ee810', + 'sha256': '49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709', # noqa + 'size': 898, + 'url': 'https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip', # noqa + } + } - with self.assertRaisesRegex(ValueError, - 'Unsupported other formats for now'): - list(self.project.releases()) + dir_paths = [] + for _release_info, _author, _release, _dir_path in actual_releases: + version = _release_info['version'] + expected_pkginfo = _project_pkginfo(_dir_path) + self.assertEquals(_release_info, expected_pkginfo) + expected_author = author(expected_pkginfo) + self.assertEqual(_author, expected_author) + expected_release = expected_releases[version] + self.assertEqual(_release, expected_release) + + self.assertTrue(version in _dir_path) + self.assertTrue(self.project_name in _dir_path) + # path still exists + self.assertTrue(os.path.exists(_dir_path)) + dir_paths.append(_dir_path) + + # Ensure uncompressed paths have been destroyed + for _dir_path in dir_paths: + # path no longer exists + self.assertFalse(os.path.exists(_dir_path)) diff --git a/swh/loader/pypi/tests/test_model_data.json b/swh/loader/pypi/tests/test_model_data.json deleted file mode 100644 index 17a02e5..0000000 --- a/swh/loader/pypi/tests/test_model_data.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "info": { - "author": "bernardfrk", - "author_email": "bernard.frk@gmail.com", - "bugtrack_url": null, - "classifiers": [], - "description": "Utitilies to use the 7xydothis APIs", - "description_content_type": null, - "docs_url": null, - "download_url": "UNKNOWN", - "downloads": { - "last_day": -1, - "last_month": -1, - "last_week": -1 - }, - "home_page": "https://github.com/frkb/7xydothis", - "keywords": null, - "license": "UNKNOWN", - "maintainer": null, - "maintainer_email": null, - "name": "7xydothis", - "package_url": "https://pypi.org/project/7xydothis/", - "platform": "UNKNOWN", - "project_url": "https://pypi.org/project/7xydothis/", - "project_urls": { - "Download": "UNKNOWN", - "Homepage": "https://github.com/frkb/7xydothis" - }, - "release_url": "https://pypi.org/project/7xydothis/0.1.1/", - "requires_dist": null, - "requires_python": null, - "summary": "Utitilies to use the 7xydothis APIs", - "version": "0.1.1" - }, - "last_serial": 2668125, - "releases": { - "0.1": [ - { - "comment_text": "", - "digests": { - "md5": "578e4bde98db732109d0698aba168a06", - "sha256": "7e6f59be532d43ac0ad32da6a068417f0973285a38a08f3f5056f79770f2f973" - }, - "downloads": -1, - "filename": "7xydothis-0.1.tar.gz", - "has_sig": false, - "md5_digest": "578e4bde98db732109d0698aba168a06", - "packagetype": "sdist", - "python_version": "source", - "requires_python": null, - "size": 868, - "upload_time": "2017-02-25T21:31:02", - "url": "https://files.pythonhosted.org/packages/68/55/6a00e46a1a10e7a0731e50cbcc9f6243c5112eeda8326d781a03a1254105/7xydothis-0.1.tar.gz" - } - ], - "0.1.1": [ - { - "comment_text": "", - "digests": { - "md5": "75fe55b933330adbde027b6edc74863d", - "sha256": "76d243b70a10d51ea87312a97a7d7b1a525984fd56d1c5f41650a1fa0fde1bc1" - }, - "downloads": -1, - "filename": "7xydothis-0.1.1.tar.gz", - "has_sig": false, - "md5_digest": "75fe55b933330adbde027b6edc74863d", - "packagetype": "sdist", - "python_version": "source", - "requires_python": null, - "size": 875, - "upload_time": "2017-02-25T21:41:37", - "url": "https://files.pythonhosted.org/packages/96/64/6fd8e189aa97820b306f06dbce02d618bf155379575c553db3d2c2eda045/7xydothis-0.1.1.tar.gz" - } - ] - }, - "urls": [ - { - "comment_text": "", - "digests": { - "md5": "75fe55b933330adbde027b6edc74863d", - "sha256": "76d243b70a10d51ea87312a97a7d7b1a525984fd56d1c5f41650a1fa0fde1bc1" - }, - "downloads": -1, - "filename": "7xydothis-0.1.1.tar.gz", - "has_sig": false, - "md5_digest": "75fe55b933330adbde027b6edc74863d", - "packagetype": "sdist", - "python_version": "source", - "requires_python": null, - "size": 875, - "upload_time": "2017-02-25T21:41:37", - "url": "https://files.pythonhosted.org/packages/96/64/6fd8e189aa97820b306f06dbce02d618bf155379575c553db3d2c2eda045/7xydothis-0.1.1.tar.gz" - } - ] -}