diff --git a/swh/loader/git/loader.py b/swh/loader/git/loader.py --- a/swh/loader/git/loader.py +++ b/swh/loader/git/loader.py @@ -259,7 +259,7 @@ """ def __init__(self, *args, **kwargs): - super().__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) self.temp_dir = self.repo_path = None def project_name_from_archive(self, archive_path): diff --git a/swh/loader/git/tests/__init__.py b/swh/loader/git/tests/__init__.py --- a/swh/loader/git/tests/__init__.py +++ b/swh/loader/git/tests/__init__.py @@ -0,0 +1,21 @@ +TEST_LOADER_CONFIG = { + 'storage': { + 'cls': 'memory', + 'args': { + } + }, + 'send_contents': True, + 'send_directories': True, + 'send_revisions': True, + 'send_releases': True, + 'send_snapshot': True, + + 'content_size_limit': 100 * 1024 * 1024, + 'content_packet_size': 10, + 'content_packet_size_bytes': 100 * 1024 * 1024, + 'directory_packet_size': 10, + 'revision_packet_size': 10, + 'release_packet_size': 10, + + 'save_data': False, +} diff --git a/swh/loader/git/tests/test_loader.py b/swh/loader/git/tests/test_loader.py --- a/swh/loader/git/tests/test_loader.py +++ b/swh/loader/git/tests/test_loader.py @@ -9,26 +9,15 @@ import subprocess from swh.loader.git.loader import GitLoader, GitLoaderFromArchive -from swh.loader.core.tests import BaseLoaderTest, LoaderNoStorage -from swh.model.hashutil import hash_to_bytes, hash_to_bytehex +from swh.loader.core.tests import BaseLoaderTest +from . import TEST_LOADER_CONFIG -class MockStorage0: - """The storage's state before anything is added.""" - def snapshot_get_latest(self, origin_id): - return None - def content_missing(self, contents, key_hash='sha1'): - return [c[key_hash] for c in contents] - - def directory_missing(self, directories): - return directories - - def revision_missing(self, revisions): - return revisions - - def object_find_by_sha1_git(self, ids): - return {} +class GitLoaderFromArchive(GitLoaderFromArchive): + def project_name_from_archive(self, archive_path): + # We don't want the project name to be 'resources'. + return 'testrepo' CONTENT1 = { @@ -70,8 +59,6 @@ }, } -SUBDIR_HASH = hash_to_bytes('d53f143d5f3aadb278aad60c4e9a17945a2d68de') - # directory hashes obtained with: # gco b6f40292c4e94a8f7e7b4aff50e6c7429ab98e2a # swh-hashtree --ignore '.git' --path . @@ -105,91 +92,6 @@ } -class MockStorage1: - """The storage's state after the first snapshot is loaded.""" - def snapshot_get_latest(self, origin_id): - # The following line reencodes SNAPSHOT1 from the format expected - # by assertSnapshotOk to the one that Storage.snapshot_get_latest - # returns. - return { - 'id': hash_to_bytes(SNAPSHOT1['id']), - 'branches': { - branch_name.encode(): { - 'target': hash_to_bytes(branch['target']), - 'target_type': branch['target_type']} - for (branch_name, branch) - in SNAPSHOT1['branches'].items()}} - - def content_missing(self, contents, key_hash='sha1'): - return map(hash_to_bytes, - {c[key_hash] for c in contents} - CONTENT1) - - def directory_missing(self, directories): - assert all(isinstance(d, bytes) for d in directories) - return (set(directories) - - set(map(hash_to_bytes, REVISIONS1)) - - {hash_to_bytes(SUBDIR_HASH)}) - - def revision_missing(self, revisions): - assert all(isinstance(r, bytes) for r in revisions) - return list(set(revisions) - set(map(hash_to_bytes, REVISIONS1))) - - def object_find_by_sha1_git(self, ids): - res = {} - for id_ in ids: - found = [] - decoded_id = hash_to_bytehex(id_) - if decoded_id in REVISIONS1: - found.append({ - 'sha1_git': id_, - 'type': 'revision', - 'id': id_, - 'object_id': 42, - }) - elif decoded_id in REVISIONS1.values(): - found.append({ - 'sha1_git': id_, - 'type': 'directory', - 'id': id_, - 'object_id': 42, - }) - elif decoded_id == SUBDIR_HASH: - found.append({ - 'sha1_git': id_, - 'type': 'directory', - 'id': id_, - 'object_id': 42, - }) - elif decoded_id in CONTENT1: - found.append({ - 'sha1_git': id_, - 'type': 'content', - 'id': id_, - 'object_id': 42, - }) - res[id_] = found - return res - - -class LoaderNoStorageMixin(LoaderNoStorage): - def __init__(self): - super().__init__() - self.origin_id = 1 - self.visit = 1 - self.storage = MockStorage0() - - -class GitLoaderNoStorage(LoaderNoStorageMixin, GitLoader): - pass - - -class GitLoaderFromArchiveNoStorage(LoaderNoStorageMixin, - GitLoaderFromArchive): - def project_name_from_archive(self, archive_path): - # We don't want the project name to be 'resources'. - return 'testrepo' - - class BaseGitLoaderTest(BaseLoaderTest): def setUp(self, archive_name, uncompress_archive, filename='testrepo'): super().setUp(archive_name=archive_name, filename=filename, @@ -198,6 +100,11 @@ uncompress_archive=uncompress_archive) +class TestGitLoader(GitLoader): + def parse_config_file(self, *args, **kwargs): + return TEST_LOADER_CONFIG + + class BaseDirGitLoaderTest(BaseGitLoaderTest): """Mixin base loader test to prepare the git repository to uncompress, load and test the results. @@ -207,7 +114,8 @@ """ def setUp(self): super().setUp('testrepo.tgz', True) - self.loader = GitLoaderNoStorage() + self.loader = TestGitLoader() + self.storage = self.loader.storage def load(self): return self.loader.load( @@ -216,6 +124,11 @@ directory=self.destination_path) +class TestGitLoaderFromArchive(GitLoaderFromArchive): + def parse_config_file(self, *args, **kwargs): + return TEST_LOADER_CONFIG + + class BaseZipGitLoaderTest(BaseGitLoaderTest): """Mixin base loader test to prepare the git repository to uncompress, load and test the results. @@ -226,7 +139,8 @@ def setUp(self): super().setUp('testrepo.tgz', True) self._setup_zip() - self.loader = GitLoaderFromArchiveNoStorage() + self.loader = TestGitLoaderFromArchive() + self.storage = self.loader.storage def _setup_zip(self): self._zip_file = tempfile.NamedTemporaryFile('ab', suffix='.zip') @@ -264,15 +178,15 @@ res = self.load() self.assertEqual(res['status'], 'eventful', res) - self.assertCountContents(4) # two README, file1, file2 + self.assertContentsContain(CONTENT1) self.assertCountDirectories(7) self.assertCountReleases(0) # FIXME: why not 2? self.assertCountRevisions(7) self.assertCountSnapshots(1) - self.assertRevisionsOk(REVISIONS1) + self.assertRevisionsContain(REVISIONS1) - self.assertSnapshotOk(SNAPSHOT1) + self.assertSnapshotEqual(SNAPSHOT1) self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.visit_status(), 'full') @@ -283,7 +197,6 @@ res = self.load() self.assertEqual(res['status'], 'eventful') - self.loader.storage = MockStorage1() res = self.load() self.assertEqual(res['status'], 'uneventful') self.assertCountSnapshots(1) @@ -335,10 +248,10 @@ self.assertCountRevisions(7 + 1) self.assertCountSnapshots(1 + 1) - self.assertRevisionsOk(revisions) + self.assertRevisionsContain(revisions) # TODO: how to check the snapshot id? - # self.assertSnapshotOk(SNAPSHOT1) + # self.assertSnapshotEqual(SNAPSHOT1) self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.visit_status(), 'full') @@ -359,10 +272,10 @@ self.assertCountRevisions(7 + 2) self.assertCountSnapshots(1 + 1 + 1) - self.assertRevisionsOk(revisions) + self.assertRevisionsContain(revisions) # TODO: how to check the snapshot id? - # self.assertSnapshotOk(SNAPSHOT1) + # self.assertSnapshotEqual(SNAPSHOT1) self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.visit_status(), 'full') diff --git a/swh/loader/git/tests/test_updater.py b/swh/loader/git/tests/test_updater.py --- a/swh/loader/git/tests/test_updater.py +++ b/swh/loader/git/tests/test_updater.py @@ -1,18 +1,21 @@ from swh.loader.git.updater import BulkUpdater -from swh.loader.git.tests.test_loader import ( - DirGitLoaderTest, LoaderNoStorageMixin) +from swh.loader.git.tests.test_loader import DirGitLoaderTest -class BulkUpdaterNoStorage(LoaderNoStorageMixin, BulkUpdater): - """Subclass of BulkUpdater that uses a mock storage.""" - pass +class TestBulkUpdater(BulkUpdater): + def parse_config_file(self, *args, **kwargs): + return { + **super().parse_config_file(*args, **kwargs), + 'storage': {'cls': 'memory', 'args': {}} + } class BulkUpdaterTest(DirGitLoaderTest): """Same tests as for the GitLoader, but running on BulkUpdater.""" def setUp(self): super().setUp() - self.loader = BulkUpdaterNoStorage() + self.loader = TestBulkUpdater() + self.storage = self.loader.storage def load(self): return self.loader.load(