diff --git a/swh/loader/pypi/tests/resources/0805nexter+new-made-up-release.json b/swh/loader/pypi/tests/resources/0805nexter+new-made-up-release.json new file mode 100644 index 0000000..fbeb488 --- /dev/null +++ b/swh/loader/pypi/tests/resources/0805nexter+new-made-up-release.json @@ -0,0 +1,114 @@ +{ + "info": { + "author": "hgtkpython", + "author_email": "2868989685@qq.com", + "bugtrack_url": null, + "classifiers": [], + "description": "UNKNOWN", + "description_content_type": null, + "docs_url": null, + "download_url": "UNKNOWN", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "http://www.hp.com", + "keywords": null, + "license": "UNKNOWN", + "maintainer": null, + "maintainer_email": null, + "name": "0805nexter", + "package_url": "https://pypi.org/project/0805nexter/", + "platform": "UNKNOWN", + "project_url": "https://pypi.org/project/0805nexter/", + "project_urls": { + "Download": "UNKNOWN", + "Homepage": "http://www.hp.com" + }, + "release_url": "https://pypi.org/project/0805nexter/1.3.0/", + "requires_dist": null, + "requires_python": null, + "summary": "a simple printer of nested lest", + "version": "1.3.0" + }, + "last_serial": 1931736, + "releases": { + "1.1.0": [ + { + "comment_text": "", + "digests": { + "md5": "07fc93fc12821c1405c3483db88154af", + "sha256": "52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035" + }, + "downloads": -1, + "filename": "0805nexter-1.1.0.zip", + "has_sig": false, + "md5_digest": "07fc93fc12821c1405c3483db88154af", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 862, + "upload_time": "2016-01-31T05:28:42", + "url": "https://files.pythonhosted.org/packages/ec/65/c0116953c9a3f47de89e71964d6c7b0c783b01f29fa3390584dbf3046b4d/0805nexter-1.1.0.zip" + } + ], + "1.2.0": [ + { + "comment_text": "", + "digests": { + "md5": "89123c78bd5d3f61cb8f46029492b18a", + "sha256": "49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709" + }, + "downloads": -1, + "filename": "0805nexter-1.2.0.zip", + "has_sig": false, + "md5_digest": "89123c78bd5d3f61cb8f46029492b18a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 898, + "upload_time": "2016-01-31T05:51:25", + "url": "https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip" + } + ], + "1.3.0": [ + { + "comment_text": "Made up release 1.3.0 for swh-loader-pypi purposes", + "digests": { + "md5": "54d9750a1ab7ab82cd8c460c2c6c0ecc", + "sha256": "7097c49fb8ec24a7aaab54c3dbfbb5a6ca1431419d9ee0f6c363d9ad01d2b8b1" + }, + "downloads": -1, + "filename": "0805nexter-1.3.0.zip", + "has_sig": false, + "md5_digest": "54d9750a1ab7ab82cd8c460c2c6c0ecc", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 1370, + "upload_time": "2018-09-17T16:18:01", + "url": "https://files.pythonhosted.org/packages/70/97/c49fb8ec24a7aaab54c3dbfbb5a6ca1431419d9ee0f6c363d9ad01d2b8b1/0805nexter-1.3.0.zip" + } + ] + }, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "89123c78bd5d3f61cb8f46029492b18a", + "sha256": "49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709" + }, + "downloads": -1, + "filename": "0805nexter-1.2.0.zip", + "has_sig": false, + "md5_digest": "89123c78bd5d3f61cb8f46029492b18a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 898, + "upload_time": "2016-01-31T05:51:25", + "url": "https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip" + } + ] +} diff --git a/swh/loader/pypi/tests/resources/0805nexter-unpublished-release.json b/swh/loader/pypi/tests/resources/0805nexter-unpublished-release.json new file mode 100644 index 0000000..c9d0197 --- /dev/null +++ b/swh/loader/pypi/tests/resources/0805nexter-unpublished-release.json @@ -0,0 +1,116 @@ +{ + "info": { + "author": "hgtkpython", + "author_email": "2868989685@qq.com", + "bugtrack_url": null, + "classifiers": [], + "description": "UNKNOWN", + "description_content_type": null, + "docs_url": null, + "download_url": "UNKNOWN", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "http://www.hp.com", + "keywords": null, + "license": "UNKNOWN", + "maintainer": null, + "maintainer_email": null, + "name": "0805nexter", + "package_url": "https://pypi.org/project/0805nexter/", + "platform": "UNKNOWN", + "project_url": "https://pypi.org/project/0805nexter/", + "project_urls": { + "Download": "UNKNOWN", + "Homepage": "http://www.hp.com" + }, + "release_url": "https://pypi.org/project/0805nexter/1.4.0/", + "requires_dist": null, + "requires_python": null, + "summary": "a simple printer of nested lest", + "version": "1.4.0" + }, + "last_serial": 1931736, + "releases": { + "1.1.0": [], + "1.2.0": [ + { + "comment_text": "", + "digests": { + "md5": "89123c78bd5d3f61cb8f46029492b18a", + "sha256": "49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709" + }, + "downloads": -1, + "filename": "0805nexter-1.2.0.zip", + "has_sig": false, + "md5_digest": "89123c78bd5d3f61cb8f46029492b18a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 898, + "upload_time": "2016-01-31T05:51:25", + "url": "https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip" + } + ], + "1.3.0": [ + { + "comment_text": "Made up release 1.3.0 for swh-loader-pypi purposes", + "digests": { + "md5": "54d9750a1ab7ab82cd8c460c2c6c0ecc", + "sha256": "7097c49fb8ec24a7aaab54c3dbfbb5a6ca1431419d9ee0f6c363d9ad01d2b8b1" + }, + "downloads": -1, + "filename": "0805nexter-1.3.0.zip", + "has_sig": false, + "md5_digest": "54d9750a1ab7ab82cd8c460c2c6c0ecc", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 1370, + "upload_time": "2018-09-17T16:18:01", + "url": "https://files.pythonhosted.org/packages/70/97/c49fb8ec24a7aaab54c3dbfbb5a6ca1431419d9ee0f6c363d9ad01d2b8b1/0805nexter-1.3.0.zip" + } + ], + "1.4.0": [ + { + "comment_text": "1.4.0: Made up release for swh-loader-pypi test purposes", + "digests": { + "md5": "a30afef2f605ea837e6d5820412d5a7b", + "sha256": "4f6e85d0f9f04c00429ddcb25530f5c125fa169bd470cda9f04daa40862eb712" + }, + "downloads": -1, + "filename": "0805nexter-1.4.0.zip", + "has_sig": false, + "md5_digest": "54d9750a1ab7ab82cd8c460c2c6c0ecc", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 1370, + "upload_time": "2018-09-18T09:18:01", + "url": "https://files.pythonhosted.org/packages/4f/6e/85d0f9f04c00429ddcb25530f5c125fa169bd470cda9f04daa40862eb712/0805nexter-1.4.0.zip" + } + ] + + }, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "89123c78bd5d3f61cb8f46029492b18a", + "sha256": "49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709" + }, + "downloads": -1, + "filename": "0805nexter-1.2.0.zip", + "has_sig": false, + "md5_digest": "89123c78bd5d3f61cb8f46029492b18a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 898, + "upload_time": "2016-01-31T05:51:25", + "url": "https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip" + } + ] +} diff --git a/swh/loader/pypi/tests/resources/0805nexter.json b/swh/loader/pypi/tests/resources/0805nexter.json index 52c8520..357bf16 100644 --- a/swh/loader/pypi/tests/resources/0805nexter.json +++ b/swh/loader/pypi/tests/resources/0805nexter.json @@ -1 +1,95 @@ -{"info":{"author":"hgtkpython","author_email":"2868989685@qq.com","bugtrack_url":null,"classifiers":[],"description":"UNKNOWN","description_content_type":null,"docs_url":null,"download_url":"UNKNOWN","downloads":{"last_day":-1,"last_month":-1,"last_week":-1},"home_page":"http://www.hp.com","keywords":null,"license":"UNKNOWN","maintainer":null,"maintainer_email":null,"name":"0805nexter","package_url":"https://pypi.org/project/0805nexter/","platform":"UNKNOWN","project_url":"https://pypi.org/project/0805nexter/","project_urls":{"Download":"UNKNOWN","Homepage":"http://www.hp.com"},"release_url":"https://pypi.org/project/0805nexter/1.2.0/","requires_dist":null,"requires_python":null,"summary":"a simple printer of nested lest","version":"1.2.0"},"last_serial":1931736,"releases":{"1.1.0":[{"comment_text":"","digests":{"md5":"07fc93fc12821c1405c3483db88154af","sha256":"52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035"},"downloads":-1,"filename":"0805nexter-1.1.0.zip","has_sig":false,"md5_digest":"07fc93fc12821c1405c3483db88154af","packagetype":"sdist","python_version":"source","requires_python":null,"size":862,"upload_time":"2016-01-31T05:28:42","url":"https://files.pythonhosted.org/packages/ec/65/c0116953c9a3f47de89e71964d6c7b0c783b01f29fa3390584dbf3046b4d/0805nexter-1.1.0.zip"}],"1.2.0":[{"comment_text":"","digests":{"md5":"89123c78bd5d3f61cb8f46029492b18a","sha256":"49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709"},"downloads":-1,"filename":"0805nexter-1.2.0.zip","has_sig":false,"md5_digest":"89123c78bd5d3f61cb8f46029492b18a","packagetype":"sdist","python_version":"source","requires_python":null,"size":898,"upload_time":"2016-01-31T05:51:25","url":"https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip"}]},"urls":[{"comment_text":"","digests":{"md5":"89123c78bd5d3f61cb8f46029492b18a","sha256":"49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709"},"downloads":-1,"filename":"0805nexter-1.2.0.zip","has_sig":false,"md5_digest":"89123c78bd5d3f61cb8f46029492b18a","packagetype":"sdist","python_version":"source","requires_python":null,"size":898,"upload_time":"2016-01-31T05:51:25","url":"https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip"}]} \ No newline at end of file +{ + "info": { + "author": "hgtkpython", + "author_email": "2868989685@qq.com", + "bugtrack_url": null, + "classifiers": [], + "description": "UNKNOWN", + "description_content_type": null, + "docs_url": null, + "download_url": "UNKNOWN", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "http://www.hp.com", + "keywords": null, + "license": "UNKNOWN", + "maintainer": null, + "maintainer_email": null, + "name": "0805nexter", + "package_url": "https://pypi.org/project/0805nexter/", + "platform": "UNKNOWN", + "project_url": "https://pypi.org/project/0805nexter/", + "project_urls": { + "Download": "UNKNOWN", + "Homepage": "http://www.hp.com" + }, + "release_url": "https://pypi.org/project/0805nexter/1.2.0/", + "requires_dist": null, + "requires_python": null, + "summary": "a simple printer of nested lest", + "version": "1.2.0" + }, + "last_serial": 1931736, + "releases": { + "1.1.0": [ + { + "comment_text": "", + "digests": { + "md5": "07fc93fc12821c1405c3483db88154af", + "sha256": "52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035" + }, + "downloads": -1, + "filename": "0805nexter-1.1.0.zip", + "has_sig": false, + "md5_digest": "07fc93fc12821c1405c3483db88154af", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 862, + "upload_time": "2016-01-31T05:28:42", + "url": "https://files.pythonhosted.org/packages/ec/65/c0116953c9a3f47de89e71964d6c7b0c783b01f29fa3390584dbf3046b4d/0805nexter-1.1.0.zip" + } + ], + "1.2.0": [ + { + "comment_text": "", + "digests": { + "md5": "89123c78bd5d3f61cb8f46029492b18a", + "sha256": "49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709" + }, + "downloads": -1, + "filename": "0805nexter-1.2.0.zip", + "has_sig": false, + "md5_digest": "89123c78bd5d3f61cb8f46029492b18a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 898, + "upload_time": "2016-01-31T05:51:25", + "url": "https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip" + } + ] + }, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "89123c78bd5d3f61cb8f46029492b18a", + "sha256": "49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709" + }, + "downloads": -1, + "filename": "0805nexter-1.2.0.zip", + "has_sig": false, + "md5_digest": "89123c78bd5d3f61cb8f46029492b18a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 898, + "upload_time": "2016-01-31T05:51:25", + "url": "https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip" + } + ] +} diff --git a/swh/loader/pypi/tests/resources/archives/0805nexter-1.1.0.zip b/swh/loader/pypi/tests/resources/archives/0805nexter-1.1.0.zip new file mode 100644 index 0000000..4b8f135 Binary files /dev/null and b/swh/loader/pypi/tests/resources/archives/0805nexter-1.1.0.zip differ diff --git a/swh/loader/pypi/tests/resources/archives/0805nexter-1.2.0.zip b/swh/loader/pypi/tests/resources/archives/0805nexter-1.2.0.zip new file mode 100644 index 0000000..8638d33 Binary files /dev/null and b/swh/loader/pypi/tests/resources/archives/0805nexter-1.2.0.zip differ diff --git a/swh/loader/pypi/tests/resources/archives/0805nexter-1.3.0.zip b/swh/loader/pypi/tests/resources/archives/0805nexter-1.3.0.zip new file mode 100644 index 0000000..3fa6c3a Binary files /dev/null and b/swh/loader/pypi/tests/resources/archives/0805nexter-1.3.0.zip differ diff --git a/swh/loader/pypi/tests/resources/archives/0805nexter-1.4.0.zip b/swh/loader/pypi/tests/resources/archives/0805nexter-1.4.0.zip new file mode 100644 index 0000000..316ced2 Binary files /dev/null and b/swh/loader/pypi/tests/resources/archives/0805nexter-1.4.0.zip differ diff --git a/swh/loader/pypi/tests/test_loader.py b/swh/loader/pypi/tests/test_loader.py index f6d5201..5c24a1d 100644 --- a/swh/loader/pypi/tests/test_loader.py +++ b/swh/loader/pypi/tests/test_loader.py @@ -1,259 +1,477 @@ # Copyright (C) 2016-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import json import shutil import tempfile from nose.plugins.attrib import attr from nose.tools import istest from unittest import TestCase from swh.model import hashutil from swh.loader.pypi.client import PyPIProject from swh.loader.pypi.loader import PyPILoader from .common import PyPIClientWithCache, RESOURCES_PATH, LoaderNoStorage class TestPyPILoader(LoaderNoStorage, PyPILoader): """Real PyPILoader for test purposes (storage and pypi interactions inhibited) """ - def __init__(self, project_name): - project_metadata_file = '%s/%s.json' % (RESOURCES_PATH, project_name) + def __init__(self, project_name, json_filename=None): + if not json_filename: # defaulting to using same name as project + json_filename = '%s.json' % project_name + project_metadata_file = '%s/%s' % (RESOURCES_PATH, json_filename) project_metadata_url = 'https://pypi.org/pypi/%s/json' % project_name with open(project_metadata_file) as f: data = json.load(f) temp_dir = tempfile.mkdtemp( dir='/tmp/', prefix='swh.loader.pypi.tests-') # Will use the pypi with cache client = PyPIClientWithCache( temp_directory=temp_dir, cache_dir=RESOURCES_PATH) super().__init__(client=client) self.project = PyPIProject( client=client, project=project_name, project_metadata_url=project_metadata_url, data=data) def prepare(self, project_name, origin_url, origin_metadata_url=None): self.project_name = project_name self.origin_url = origin_url self.origin_metadata_url = origin_metadata_url self.visit = 1 # first visit self._prepare_state() @attr('fs') class BaseLoaderITest(TestCase): """Loader Test Mixin to prepare the pypi to 'load' in a test context. In this setup, the loader uses the cache to load data so no network interaction (no storage, no pypi). """ def setUp(self, project_name='0805nexter', dummy_pypi_instance='https://dummy.org'): self.tmp_root_path = tempfile.mkdtemp() - self.loader = PyPILoaderNoSnapshot(project_name=project_name) self._project = project_name self._origin_url = '%s/pypi/%s/' % (dummy_pypi_instance, project_name) self._project_metadata_url = '%s/pypi/%s/json' % ( dummy_pypi_instance, project_name) def tearDown(self): shutil.rmtree(self.tmp_root_path) def assertContentsOk(self, expected_contents): contents = self.loader.all_contents self.assertEquals(len(contents), len(expected_contents)) for content in contents: content_id = hashutil.hash_to_hex(content['sha1']) self.assertIn(content_id, expected_contents) def assertDirectoriesOk(self, expected_directories): directories = self.loader.all_directories self.assertEquals(len(directories), len(expected_directories)) for _dir in directories: _dir_id = hashutil.hash_to_hex(_dir['id']) self.assertIn(_dir_id, expected_directories) def assertSnapshotOk(self, expected_snapshot, expected_revisions): snapshots = self.loader.all_snapshots self.assertEqual(len(snapshots), 1) snap = snapshots[0] snap_id = hashutil.hash_to_hex(snap['id']) self.assertEqual(snap_id, expected_snapshot) branches = snap['branches'] self.assertEqual(len(expected_revisions), len(branches)) for branch, target in branches.items(): rev_id = hashutil.hash_to_hex(target['target']) self.assertIn(rev_id, expected_revisions) self.assertEqual('revision', target['target_type']) def assertRevisionsOk(self, expected_revisions): # noqa: N802 """Check the loader's revisions match the expected revisions. Expects self.loader to be instantiated and ready to be inspected (meaning the loading took place). Args: expected_revisions (dict): Dict with key revision id, value the targeted directory id. """ # The last revision being the one used later to start back from for rev in self.loader.all_revisions: rev_id = hashutil.hash_to_hex(rev['id']) directory_id = hashutil.hash_to_hex(rev['directory']) self.assertEquals(expected_revisions[rev_id], directory_id) # Define loaders with no storage # They'll just accumulate the data in place # Only for testing purposes. class PyPILoaderNoSnapshot(TestPyPILoader): """Same as TestPyPILoader with no prior snapshot seen """ def _last_snapshot(self): return None class LoaderITest(BaseLoaderITest): def setUp(self, project_name='0805nexter', dummy_pypi_instance='https://dummy.org'): super().setUp(project_name, dummy_pypi_instance) self.loader = PyPILoaderNoSnapshot(project_name=project_name) @istest def load(self): """Load a pypi origin """ # when self.loader.load( self._project, self._origin_url, self._project_metadata_url) # then self.assertEquals(len(self.loader.all_contents), 6, '3 contents per release artifact files (2)') self.assertEquals(len(self.loader.all_directories), 4) self.assertEquals(len(self.loader.all_revisions), 2, '2 releases so 2 revisions should be created') self.assertEquals(len(self.loader.all_releases), 0, 'No release is created in the pypi loader') self.assertEquals(len(self.loader.all_snapshots), 1, 'Only 1 snapshot targetting all revisions') expected_contents = [ 'a61e24cdfdab3bb7817f6be85d37a3e666b34566', '938c33483285fd8ad57f15497f538320df82aeb8', 'a27576d60e08c94a05006d2e6d540c0fdb5f38c8', '405859113963cb7a797642b45f171d6360425d16', 'e5686aa568fdb1d19d7f1329267082fe40482d31', '83ecf6ec1114fd260ca7a833a2d165e71258c338', ] self.assertContentsOk(expected_contents) expected_directories = [ '05219ba38bc542d4345d5638af1ed56c7d43ca7d', 'cf019eb456cf6f78d8c4674596f1c9a97ece8f44', 'b178b66bd22383d5f16f4f5c923d39ca798861b4', 'c3a58f8b57433a4b56caaa5033ae2e0931405338', ] self.assertDirectoriesOk(expected_directories) # {revision hash: directory hash} expected_revisions = { '4c99891f93b81450385777235a37b5e966dd1571': '05219ba38bc542d4345d5638af1ed56c7d43ca7d', # noqa 'e445da4da22b31bfebb6ffc4383dbf839a074d21': 'b178b66bd22383d5f16f4f5c923d39ca798861b4', # noqa } self.assertRevisionsOk(expected_revisions) self.assertSnapshotOk('f456b03e8bf1920d64b00df234b1efedc25b6c93', expected_revisions) class PyPILoaderWithSnapshot(TestPyPILoader): - """Same as TestPyPILoader with no prior snapshot seen + """This loader provides a snapshot and lists corresponding seen + release artifacts. """ def _last_snapshot(self): + """Return last visited snapshot""" return { 'id': b'\xf4V\xb0>\x8b\xf1\x92\rd\xb0\r\xf24\xb1\xef\xed\xc2[l\x93', # noqa 'branches': { b'0805nexter-1.1.0.zip': { 'target': b'L\x99\x89\x1f\x93\xb8\x14P' b'8Ww#Z7\xb5\xe9f\xdd\x15q', 'target_type': 'revision' }, b'0805nexter-1.2.0.zip': { 'target': b'\xe4E\xdaM\xa2+1\xbf' b'\xeb\xb6\xff\xc48=\xbf\x83' b'\x9a\x07M!', 'target_type': 'revision' }, }, } def _known_artifacts(self, last_snapshot): + """List corresponding seen release artifacts""" yield from [ ( '0805nexter-1.1.0.zip', '52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035' # noqa ), ( '0805nexter-1.2.0.zip', '49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709' # noqa ) ] -class LoaderWithOriginAlreadySeenITest(BaseLoaderITest): +class LoaderNoNewChangesSinceLastVisitITest(BaseLoaderITest): + """This scenario makes use of the incremental nature of the loader. + + If nothing changes in between visits, the snapshot for the visit + must stay the same as the first visit. + + """ def setUp(self, project_name='0805nexter', dummy_pypi_instance='https://dummy.org'): super().setUp(project_name, dummy_pypi_instance) self.loader = PyPILoaderWithSnapshot(project_name=project_name) @istest def load(self): - """Load a pypi origin already injected will result with only 1 snapshot + """Load a PyPI origin without new changes results in 1 same snapshot """ # when self.loader.load( self._project, self._origin_url, self._project_metadata_url) # then self.assertEquals(len(self.loader.all_contents), 0) self.assertEquals(len(self.loader.all_directories), 0) self.assertEquals(len(self.loader.all_revisions), 0) self.assertEquals(len(self.loader.all_releases), 0) self.assertEquals(len(self.loader.all_snapshots), 1) self.assertContentsOk([]) self.assertDirectoriesOk([]) self.assertRevisionsOk(expected_revisions={}) expected_revisions = { '4c99891f93b81450385777235a37b5e966dd1571': '05219ba38bc542d4345d5638af1ed56c7d43ca7d', # noqa 'e445da4da22b31bfebb6ffc4383dbf839a074d21': 'b178b66bd22383d5f16f4f5c923d39ca798861b4', # noqa } + expected_snapshot_id = 'f456b03e8bf1920d64b00df234b1efedc25b6c93' self.assertSnapshotOk('f456b03e8bf1920d64b00df234b1efedc25b6c93', expected_revisions) + + _id = hashutil.hash_to_hex(self.loader._last_snapshot()['id']) + self.assertEquals(expected_snapshot_id, _id) + + +class LoaderNewChangesSinceLastVisitITest(BaseLoaderITest): + """In this scenario, a visit has already taken place. + An existing snapshot exists. + + This time, the PyPI project has changed, a new release (with 1 new + release artifact) has been uploaded. The old releases did not + change. + + The visit results in a new snapshot. + + The new snapshot shares the same history as prior visit's + snapshot. It holds a new branch targetting the new revision. + + """ + def setUp(self, project_name='0805nexter', + dummy_pypi_instance='https://dummy.org'): + super().setUp(project_name, dummy_pypi_instance) + self.loader = PyPILoaderWithSnapshot( + project_name=project_name, + json_filename='0805nexter+new-made-up-release.json') + + @istest + def load(self): + """Load a PyPI origin with changes results in 1 new snapshot + + """ + # when + self.loader.load( + self._project, self._origin_url, self._project_metadata_url) + + # then + self.assertEquals( + len(self.loader.all_contents), 4, + "3 + 1 new content (only change between 1.2.0 and 1.3.0 archives)") + self.assertEquals(len(self.loader.all_directories), 2) + self.assertEquals( + len(self.loader.all_revisions), 1, + "This results in 1 new revision targetting that new directory id") + self.assertEquals(len(self.loader.all_releases), 0) + self.assertEquals(len(self.loader.all_snapshots), 1) + + expected_contents = [ + '92689fa2b7fb4d4fc6fb195bf73a50c87c030639', # new one + '405859113963cb7a797642b45f171d6360425d16', + '83ecf6ec1114fd260ca7a833a2d165e71258c338', + 'e5686aa568fdb1d19d7f1329267082fe40482d31', + ] + + self.assertContentsOk(expected_contents) + + expected_directories = [ + 'e226e7e4ad03b4fc1403d69a18ebdd6f2edd2b3a', + '52604d46843b898f5a43208045d09fcf8731631b', + ] + self.assertDirectoriesOk(expected_directories) + + expected_revisions = { + 'fb46e49605b0bbe69f8c53d315e89370e7c6cb5d': 'e226e7e4ad03b4fc1403d69a18ebdd6f2edd2b3a', # noqa + } + self.assertRevisionsOk(expected_revisions) + + old_revisions = { + '4c99891f93b81450385777235a37b5e966dd1571': '05219ba38bc542d4345d5638af1ed56c7d43ca7d', # noqa + 'e445da4da22b31bfebb6ffc4383dbf839a074d21': 'b178b66bd22383d5f16f4f5c923d39ca798861b4', # noqa + } + for rev, dir_id in old_revisions.items(): + expected_revisions[rev] = dir_id + + expected_snapshot_id = 'e5beda90e9ddbc8672734ed172246b06fcbc6827' + self.assertSnapshotOk(expected_snapshot_id, expected_revisions) + + _id = hashutil.hash_to_hex(self.loader._last_snapshot()['id']) + self.assertNotEqual(expected_snapshot_id, _id) + + +class PyPILoaderWithSnapshot2(TestPyPILoader): + """This loader provides a snapshot and lists corresponding seen + release artifacts. + + """ + def _last_snapshot(self): + """Return last visited snapshot""" + return { + 'id': b"\xe5\xbe\xda\x90\xe9\xdd\xbc\x86rsN\xd1r$k\x06\xfc\xbch'", + 'branches': { + b'0805nexter-1.1.0.zip': { + 'target': b'L\x99\x89\x1f\x93\xb8\x14P8Ww#Z7\xb5\xe9f\xdd\x15q', # noqa + 'target_type': 'revision' + }, + b'0805nexter-1.2.0.zip': { + 'target': b'\xe4E\xdaM\xa2+1\xbf\xeb\xb6\xff\xc48=\xbf\x83\x9a\x07M!', # noqa + 'target_type': 'revision' + }, + b'0805nexter-1.3.0.zip': { + 'target': b'\xfbF\xe4\x96\x05\xb0\xbb\xe6\x9f\x8cS\xd3\x15\xe8\x93p\xe7\xc6\xcb]', # noqa + 'target_type': 'revision' + } + } + } + + def _known_artifacts(self, last_snapshot): + """List corresponding seen release artifacts""" + return [ + ( + '0805nexter-1.1.0.zip', + '52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035' # noqa + ), + ( + '0805nexter-1.2.0.zip', + '49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709' # noqa + ), + ( + '0805nexter-1.3.0.zip', + '7097c49fb8ec24a7aaab54c3dbfbb5a6ca1431419d9ee0f6c363d9ad01d2b8b1' # noqa + ) + ] + + +class LoaderChangesOldReleaseArtifactRemovedSinceLastVisit(BaseLoaderITest): + """In this scenario, a visit has already taken place. An existing + snapshot exists. + + The PyPI project has changed: + - a new release has been uploaded + - an older one has been removed + + # What wrongly happens now + + The visit results in a new snapshot, such snapshot shares the same + branches as last visit's snapshot (including the unpublished + release artifacts). + + # What should happen: + + The visit should result in a new snapshot. Such snapshot shares some of + the same branches as prior visit (but not all): + + - new release artifact branch exists + - old release artifact branch has been removed + - the other unchanged release artifact branches are left unchanged + + """ + def setUp(self, project_name='0805nexter', + dummy_pypi_instance='https://dummy.org'): + super().setUp(project_name, dummy_pypi_instance) + self.loader = PyPILoaderWithSnapshot2( + project_name=project_name, + json_filename='0805nexter-unpublished-release.json') + + @istest + def load(self): + """Load PyPI origin with removed artifact + changes ~> 1 new snapshot + + """ + # when + self.loader.load( + self._project, self._origin_url, self._project_metadata_url) + + # then + self.assertEquals( + len(self.loader.all_contents), 4, + "3 + 1 new content (only change between 1.3.0 and 1.4.0 archives)") + self.assertEquals(len(self.loader.all_directories), 2) + self.assertEquals( + len(self.loader.all_revisions), 1, + "This results in 1 new revision targetting that new directory id") + self.assertEquals(len(self.loader.all_releases), 0) + self.assertEquals(len(self.loader.all_snapshots), 1) + + expected_contents = [ + 'e2d68a197e3a3ad0fc6de28749077892c2148043', # new one + '405859113963cb7a797642b45f171d6360425d16', + '83ecf6ec1114fd260ca7a833a2d165e71258c338', + 'e5686aa568fdb1d19d7f1329267082fe40482d31', + ] + + self.assertContentsOk(expected_contents) + + expected_directories = [ + 'a2b7621f3e52eb3632657f6e3436bd08202db56f', # new one + '770e21215ecac53cea331d8ea4dc0ffc9d979367', + ] + self.assertDirectoriesOk(expected_directories) + + expected_revisions = { + '5e91875f096ac48c98d74acf307439a3490f2827': '770e21215ecac53cea331d8ea4dc0ffc9d979367', # noqa + } + self.assertRevisionsOk(expected_revisions) + + old_revisions = { + '4c99891f93b81450385777235a37b5e966dd1571': '05219ba38bc542d4345d5638af1ed56c7d43ca7d', # noqa + 'e445da4da22b31bfebb6ffc4383dbf839a074d21': 'b178b66bd22383d5f16f4f5c923d39ca798861b4', # noqa + 'fb46e49605b0bbe69f8c53d315e89370e7c6cb5d': 'e226e7e4ad03b4fc1403d69a18ebdd6f2edd2b3a', # noqa + } + for rev, dir_id in old_revisions.items(): + expected_revisions[rev] = dir_id + + expected_snapshot_id = '0f36392a2c825bafa766b371f2485f9549eec41c' + self.assertSnapshotOk(expected_snapshot_id, expected_revisions) + + _id = hashutil.hash_to_hex(self.loader._last_snapshot()['id']) + self.assertNotEqual(expected_snapshot_id, _id)