diff --git a/debian/control b/debian/control --- a/debian/control +++ b/debian/control @@ -15,9 +15,9 @@ patool, python3-swh.core (>= 0.0.36~), python3-swh.model (>= 0.0.27~), - python3-swh.storage (>= 0.0.95~), + python3-swh.storage (>= 0.0.114~), python3-swh.scheduler (>= 0.0.19~), - python3-swh.loader.core (>= 0.0.35~), + python3-swh.loader.core (>= 0.0.36~), python-sqlitedict Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/source/swh-loader-mercurial/ @@ -25,9 +25,9 @@ Package: python3-swh.loader.mercurial Architecture: all Depends: python3-swh.core (>= 0.0.36~), - python3-swh.loader.core (>= 0.0.35~), + python3-swh.loader.core (>= 0.0.36~), python3-swh.model (>= 0.0.27~), - python3-swh.storage (>= 0.0.95~), + python3-swh.storage (>= 0.0.114~), python3-swh.scheduler (>= 0.0.19~), patool, python-sqlitedict, diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,5 +1,5 @@ swh.core >= 0.0.36 swh.model >= 0.0.27 -swh.storage >= 0.0.95 +swh.storage >= 0.0.114 swh.scheduler >= 0.0.19 -swh.loader.core >= 0.0.35 +swh.loader.core >= 0.0.36 diff --git a/swh/loader/mercurial/tests/test_loader.py b/swh/loader/mercurial/tests/test_loader.py --- a/swh/loader/mercurial/tests/test_loader.py +++ b/swh/loader/mercurial/tests/test_loader.py @@ -5,30 +5,54 @@ import os -from swh.loader.core.tests import BaseLoaderTest, LoaderNoStorage +from swh.loader.core.tests import BaseLoaderTest from swh.loader.mercurial.loader import HgBundle20Loader +_LOADER_TEST_CONFIG = { + 'bundle_filename': 'HG20_none_bundle', + 'cache1_size': 838860800, + 'cache2_size': 838860800, + 'content_packet_size': 100000, + 'content_packet_size_bytes': 1073741824, + 'content_size_limit': 104857600, + 'directory_packet_size': 25000, + 'log_db': 'dbname=softwareheritage-log', + 'occurrence_packet_size': 100000, + 'reduce_effort': False, + 'release_packet_size': 100000, + 'revision_packet_size': 100000, + 'save_data': False, + 'save_data_path': '', + 'send_contents': True, + 'send_directories': True, + 'send_occurrences': True, + 'send_releases': True, + 'send_revisions': True, + 'send_snapshot': True, + 'storage': {'args': {}, 'cls': 'memory'}, + 'temp_directory': '/tmp/swh.loader.mercurial' +} + + +class HgLoaderMemoryStorage(HgBundle20Loader): + """The mercurial loader to test. -class MockStorage: - """A mixin inhibited storage overriding *_missing methods. Those are - called from within the mercurial loader. + Its behavior has been changed to: + - not use any persistence (no storage, or for now a passthrough + storage with no filtering) + - not use the default configuration loading - Rationale: Need to take control of the current behavior prior - to refactor it. The end game is to remove this when we will - have tests ok. + At the end of the tests, you can make sure you have the rights + objects. """ - def content_missing(self, contents, key_hash='sha1'): - return [c[key_hash] for c in contents] - - def directory_missing(self, directories): - return directories - - def release_missing(self, releases): - return releases + def __init__(self): + super().__init__() + self.origin_id = 1 + self.visit = 1 - def revision_missing(self, revisions): - return revisions + def parse_config_file(self, *args, **kwargs): + return _LOADER_TEST_CONFIG class BaseHgLoaderTest(BaseLoaderTest): @@ -42,45 +66,14 @@ super().setUp(archive_name=archive_name, filename=filename, prefix_tmp_folder_name='swh.loader.mercurial.', start_path=os.path.dirname(__file__)) - - -class HgLoaderNoStorage(LoaderNoStorage, HgBundle20Loader): - """The mercurial loader to test. - - Its behavior has been changed to: - - not use any persistence (no storage, or for now a passthrough - storage with no filtering) - - not use the default configuration loading - - At the end of the tests, you can make sure you have the rights - objects. - - """ - ADDITIONAL_CONFIG = { - 'reduce_effort': ('bool', False), # FIXME: This needs to be - # checked (in production - # for now, this is not - # deployed.) - 'temp_directory': ('str', '/tmp/swh.loader.mercurial'), - 'cache1_size': ('int', 800*1024*1024), - 'cache2_size': ('int', 800*1024*1024), - 'bundle_filename': ('str', 'HG20_none_bundle'), - } - - def __init__(self): - super().__init__() - self.origin_id = 1 - self.visit = 1 - self.storage = MockStorage() + self.loader = HgLoaderMemoryStorage() + self.storage = self.loader.storage class LoaderITest1(BaseHgLoaderTest): """Load a mercurial repository without release """ - def setUp(self): - super().setUp() - self.loader = HgLoaderNoStorage() def test_load(self): """Load a repository with multiple branches results in 1 snapshot @@ -166,7 +159,7 @@ tip_revision_develop: directory_hash, } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) self.assertCountSnapshots(1) expected_snapshot = { @@ -187,7 +180,7 @@ } } - self.assertSnapshotOk(expected_snapshot) + self.assertSnapshotEqual(expected_snapshot) self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.visit_status(), 'full') @@ -198,7 +191,6 @@ """ def setUp(self): super().setUp(archive_name='hello.tgz', filename='hello') - self.loader = HgLoaderNoStorage() def test_load(self): """Load a repository with tags results in 1 snapshot @@ -217,7 +209,7 @@ self.assertCountRevisions(3) tip_release = '515c4d72e089404356d0f4b39d60f948b8999140' - self.assertReleasesOk([tip_release]) + self.assertReleasesContain([tip_release]) tip_revision_default = 'c3dbe4fbeaaa98dd961834e4007edb3efb0e2a27' # cf. test_loader.org for explaining from where those hashes @@ -229,7 +221,7 @@ tip_revision_default: '8f2be433c945384c85920a8e60f2a68d2c0f20fb', } - self.assertRevisionsOk(expected_revisions) + self.assertRevisionsContain(expected_revisions) self.assertCountSnapshots(1) expected_snapshot = { @@ -250,6 +242,6 @@ } } - self.assertSnapshotOk(expected_snapshot) + self.assertSnapshotEqual(expected_snapshot) self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.visit_status(), 'full')