diff --git a/debian/control b/debian/control --- a/debian/control +++ b/debian/control @@ -11,9 +11,9 @@ python3-requests, python3-setuptools, python3-swh.core, - python3-swh.loader.core (>= 0.0.35~), + python3-swh.loader.core (>= 0.0.36~), python3-swh.model (>= 0.0.28~), - python3-swh.storage (>= 0.0.108~), + python3-swh.storage (>= 0.0.114~), python3-swh.scheduler, python3-vcversioner Standards-Version: 3.9.6 @@ -22,8 +22,8 @@ Package: python3-swh.loader.pypi Architecture: all Depends: python3-swh.core, - python3-swh.loader.core (>= 0.0.35~), + python3-swh.loader.core (>= 0.0.36~), python3-swh.model (>= 0.0.28~), - python3-swh.storage (>= 0.0.108~), + python3-swh.storage (>= 0.0.114~), ${misc:Depends}, ${python3:Depends} Description: Software Heritage PyPI Loader diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,5 +1,5 @@ swh.core swh.model >= 0.0.28 -swh.storage >= 0.0.108 +swh.storage >= 0.0.114 swh.scheduler -swh.loader.core >= 0.0.35 +swh.loader.core >= 0.0.36 diff --git a/swh/loader/pypi/tests/test_loader.py b/swh/loader/pypi/tests/test_loader.py --- a/swh/loader/pypi/tests/test_loader.py +++ b/swh/loader/pypi/tests/test_loader.py @@ -8,7 +8,7 @@ import pytest -from swh.loader.core.tests import BaseLoaderTest, LoaderNoStorage +from swh.loader.core.tests import BaseLoaderTest from swh.loader.pypi.client import PyPIProject from swh.loader.pypi.loader import PyPILoader from swh.model import hashutil @@ -16,7 +16,31 @@ from .common import RESOURCES_PATH, PyPIClientWithCache -class TestPyPILoader(LoaderNoStorage, PyPILoader): +_LOADER_TESTS_CONFIG = { + 'cache': False, + 'cache_dir': '', + 'content_packet_size': 10000, + 'content_packet_size_bytes': 104857600, + 'content_size_limit': 104857600, + 'debug': False, + 'directory_packet_size': 25000, + 'log_db': 'dbname=softwareheritage-log', + 'occurrence_packet_size': 100000, + 'release_packet_size': 100000, + 'revision_packet_size': 100000, + 'save_data': False, + 'save_data_path': '', + 'send_contents': True, + 'send_directories': True, + 'send_releases': True, + 'send_revisions': True, + 'send_snapshot': True, + 'storage': {'args': {}, 'cls': 'memory'}, + 'temp_directory': '/tmp/swh.loader.pypi/' +} + + +class PyPILoaderTest(PyPILoader): """Real PyPILoader for test purposes (storage and pypi interactions inhibited) @@ -41,6 +65,9 @@ project_metadata_url=project_metadata_url, data=data) + def parse_config_file(self, *args, **kwargs): + return _LOADER_TESTS_CONFIG + def prepare(self, project_name, origin_url, origin_metadata_url=None): self.project_name = project_name @@ -68,8 +95,8 @@ dummy_pypi_instance, project_name) -class PyPILoaderNoSnapshot(TestPyPILoader): - """Same as TestPyPILoader with no prior snapshot seen +class PyPILoaderNoSnapshot(PyPILoaderTest): + """Same as PyPILoaderTest with no prior snapshot seen """ def _last_snapshot(self): @@ -81,6 +108,7 @@ dummy_pypi_instance='https://dummy.org'): super().setUp(project_name, dummy_pypi_instance) self.loader = PyPILoaderNoSnapshot(project_name=project_name) + self.storage = self.loader.storage def test_load(self): """Load a pypi origin @@ -108,7 +136,7 @@ '83ecf6ec1114fd260ca7a833a2d165e71258c338', ] - self.assertContentsOk(expected_contents) + self.assertContentsContain(expected_contents) expected_directories = [ '05219ba38bc542d4345d5638af1ed56c7d43ca7d', @@ -116,14 +144,14 @@ 'b178b66bd22383d5f16f4f5c923d39ca798861b4', 'c3a58f8b57433a4b56caaa5033ae2e0931405338', ] - self.assertDirectoriesOk(expected_directories) + self.assertDirectoriesContain(expected_directories) # {revision hash: directory hash} expected_revisions = { '4c99891f93b81450385777235a37b5e966dd1571': '05219ba38bc542d4345d5638af1ed56c7d43ca7d', # noqa 'e445da4da22b31bfebb6ffc4383dbf839a074d21': 'b178b66bd22383d5f16f4f5c923d39ca798861b4', # noqa } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) expected_branches = { 'releases/1.1.0': { @@ -140,14 +168,14 @@ }, } - self.assertSnapshotOk('ba6e158ada75d0b3cfb209ffdf6daa4ed34a227a', - expected_branches) + self.assertSnapshotEqual('ba6e158ada75d0b3cfb209ffdf6daa4ed34a227a', + expected_branches) self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.visit_status(), 'full') -class PyPILoaderWithSnapshot(TestPyPILoader): +class PyPILoaderWithSnapshot(PyPILoaderTest): """This loader provides a snapshot and lists corresponding seen release artifacts. @@ -200,6 +228,7 @@ dummy_pypi_instance='https://dummy.org'): super().setUp(project_name, dummy_pypi_instance) self.loader = PyPILoaderWithSnapshot(project_name=project_name) + self.storage = self.loader.storage def test_load(self): """Load a PyPI origin without new changes results in 1 same snapshot @@ -216,9 +245,9 @@ self.assertCountReleases(0) self.assertCountSnapshots(1) - self.assertContentsOk([]) - self.assertDirectoriesOk([]) - self.assertRevisionsOk(expected_revisions={}) + self.assertContentsContain([]) + self.assertDirectoriesContain([]) + self.assertRevisionsContain(expected_revisions={}) expected_snapshot_id = 'ba6e158ada75d0b3cfb209ffdf6daa4ed34a227a' expected_branches = { @@ -235,7 +264,7 @@ 'target_type': 'alias', }, } - self.assertSnapshotOk(expected_snapshot_id, expected_branches) + self.assertSnapshotEqual(expected_snapshot_id, expected_branches) _id = hashutil.hash_to_hex(self.loader._last_snapshot()['id']) self.assertEqual(expected_snapshot_id, _id) @@ -264,6 +293,7 @@ self.loader = PyPILoaderWithSnapshot( project_name=project_name, json_filename='0805nexter+new-made-up-release.json') + self.storage = self.loader.storage def test_load(self): """Load a PyPI origin with changes results in 1 new snapshot @@ -290,18 +320,18 @@ 'e5686aa568fdb1d19d7f1329267082fe40482d31', ] - self.assertContentsOk(expected_contents) + self.assertContentsContain(expected_contents) expected_directories = [ 'e226e7e4ad03b4fc1403d69a18ebdd6f2edd2b3a', '52604d46843b898f5a43208045d09fcf8731631b', ] - self.assertDirectoriesOk(expected_directories) + self.assertDirectoriesContain(expected_directories) expected_revisions = { 'fb46e49605b0bbe69f8c53d315e89370e7c6cb5d': 'e226e7e4ad03b4fc1403d69a18ebdd6f2edd2b3a', # noqa } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) old_revisions = { '4c99891f93b81450385777235a37b5e966dd1571': '05219ba38bc542d4345d5638af1ed56c7d43ca7d', # noqa @@ -330,7 +360,7 @@ }, } - self.assertSnapshotOk(expected_snapshot_id, expected_branches) + self.assertSnapshotEqual(expected_snapshot_id, expected_branches) _id = hashutil.hash_to_hex(self.loader._last_snapshot()['id']) self.assertNotEqual(expected_snapshot_id, _id) @@ -339,7 +369,7 @@ self.assertEqual(self.loader.visit_status(), 'full') -class PyPILoaderWithSnapshot2(TestPyPILoader): +class PyPILoaderWithSnapshot2(PyPILoaderTest): """This loader provides a snapshot and lists corresponding seen release artifacts. @@ -408,6 +438,7 @@ self.loader = PyPILoaderWithSnapshot2( project_name=project_name, json_filename='0805nexter-unpublished-release.json') + self.storage = self.loader.storage def test_load(self): """Load PyPI origin with removed artifact + changes ~> 1 new snapshot @@ -435,19 +466,19 @@ 'e5686aa568fdb1d19d7f1329267082fe40482d31', ] - self.assertContentsOk(expected_contents) + self.assertContentsContain(expected_contents) expected_directories = [ 'a2b7621f3e52eb3632657f6e3436bd08202db56f', # new one '770e21215ecac53cea331d8ea4dc0ffc9d979367', ] - self.assertDirectoriesOk(expected_directories) + self.assertDirectoriesContain(expected_directories) expected_revisions = { # 1.4.0 '5e91875f096ac48c98d74acf307439a3490f2827': '770e21215ecac53cea331d8ea4dc0ffc9d979367', # noqa } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) expected_snapshot_id = 'bb0b0c29040678eadb6dae9e43e496cc860123e4' expected_branches = { @@ -468,7 +499,7 @@ 'target_type': 'alias', }, } - self.assertSnapshotOk(expected_snapshot_id, expected_branches) + self.assertSnapshotEqual(expected_snapshot_id, expected_branches) _id = hashutil.hash_to_hex(self.loader._last_snapshot()['id']) self.assertNotEqual(expected_snapshot_id, _id)