diff --git a/swh/loader/core/tests/test_loader.py b/swh/loader/core/tests/test_loader.py --- a/swh/loader/core/tests/test_loader.py +++ b/swh/loader/core/tests/test_loader.py @@ -6,15 +6,63 @@ import datetime from swh.model.hashutil import hash_to_bytes -from swh.storage.in_memory import Storage + +from swh.loader.core.loader import SWHLoader from . import BaseLoaderTest +class DummyLoader(SWHLoader): + def cleanup(self): + pass + + def prepare(self): + pass + + def fetch_data(self): + pass + + def store_data(self): + pass + + def prepare_origin_visit(self): + origin = self.storage.origin_get( + self._test_prepare_origin_visit_data['origin']) + self.origin = origin + self.origin_id = origin['id'] + self.origin_url = origin['url'] + self.visit_date = datetime.datetime.utcnow() + self.storage.origin_visit_add(origin['id'], self.visit_date) + + def parse_config_file(self, *args, **kwargs): + return { + 'storage': { + 'cls': 'memory', + 'args': { + } + }, + + 'send_contents': True, + 'send_directories': True, + 'send_revisions': True, + 'send_releases': True, + 'send_snapshot': True, + + 'content_packet_size': 2, + 'content_packet_size_bytes': 8, + 'directory_packet_size': 2, + 'revision_packet_size': 2, + 'release_packet_size': 2, + + 'content_size_limit': 10000, + } + + class DummyBaseLoaderTest(BaseLoaderTest): def setUp(self): + self.loader = DummyLoader(logging_class='dummyloader') # do not call voluntarily super().setUp() - self.storage = Storage() + self.storage = self.loader.storage contents = [ { 'id': '34973274ccef6ab4dfaaf86599792fa9c3fe4689', @@ -24,6 +72,7 @@ 'blake2s256': b'qux1', 'status': 'visible', 'data': b'data1', + 'length': 5, }, { 'id': '61c2b3a30496d329e21af70dd2d7e097046d07b7', @@ -33,6 +82,7 @@ 'blake2s256': b'qux2', 'status': 'visible', 'data': b'data2', + 'length': 5, }, ] self.expected_contents = [content['id'] for content in contents] @@ -52,12 +102,20 @@ 'id': b'rev1', 'date': None, }, + { + 'id': b'rev2', + 'date': None, + }, ] self.in_releases = [ { 'id': b'rel1', 'date': None, }, + { + 'id': b'rel2', + 'date': None, + }, ] self.in_origins = [ { @@ -70,15 +128,19 @@ 'branches': {}, } + self.storage.origin_add(self.in_origins) + + # used by prepare_origin_visit() when it gets called + self.loader._test_prepare_origin_visit_data = { + 'origin': self.in_origins[0], + } + def tearDown(self): # do not call voluntarily super().tearDown() pass -class LoadTest1(DummyBaseLoaderTest): - def setUp(self): - super().setUp() - +class CoreLoaderTest(DummyBaseLoaderTest): def test_stateful_loader(self): """Stateful loader accumulates in place the sent data @@ -86,14 +148,27 @@ another story. """ - self.storage.directory_add(self.in_directories) - self.storage.revision_add(self.in_revisions) - self.storage.release_add(self.in_releases) + self.loader.load() # initialize the loader + + self.loader.maybe_load_contents(self.in_contents[0:1]) + self.loader.maybe_load_directories(self.in_directories[0:1]) + self.loader.maybe_load_revisions(self.in_revisions[0:1]) + self.loader.maybe_load_releases(self.in_releases[0:1]) self.assertCountContents(0) + self.assertCountDirectories(0) + self.assertCountRevisions(0) + self.assertCountReleases(0) + + self.loader.maybe_load_contents(self.in_contents[1:]) + self.loader.maybe_load_directories(self.in_directories[1:]) + self.loader.maybe_load_revisions(self.in_revisions[1:]) + self.loader.maybe_load_releases(self.in_releases[1:]) + + self.assertCountContents(len(self.in_contents)) self.assertCountDirectories(len(self.in_directories)) self.assertCountRevisions(len(self.in_revisions)) - self.assertCountSnapshots(0) + self.assertCountReleases(len(self.in_releases)) def test_stateless_loader(self): """Stateless loader accumulates in place the sent data as well @@ -102,50 +177,80 @@ another story. """ - (origin,) = self.storage.origin_add(self.in_origins) - visit = self.storage.origin_visit_add( - origin['id'], datetime.datetime.utcnow()) - self.storage.content_add(self.in_contents) - self.storage.snapshot_add(origin, visit['visit'], self.in_snapshot) + self.loader.load() # initialize the loader + + self.loader.send_contents(self.in_contents[0:1]) + self.loader.send_directories(self.in_directories[0:1]) + self.loader.send_revisions(self.in_revisions[0:1]) + self.loader.send_releases(self.in_releases[0:1]) + + self.assertCountContents(1) + self.assertCountDirectories(1) + self.assertCountRevisions(1) + self.assertCountReleases(1) + + self.loader.send_contents(self.in_contents[1:]) + self.loader.send_directories(self.in_directories[1:]) + self.loader.send_revisions(self.in_revisions[1:]) + self.loader.send_releases(self.in_releases[1:]) self.assertCountContents(len(self.in_contents)) - self.assertCountDirectories(0) - self.assertCountRevisions(0) - self.assertCountReleases(0) - self.assertCountSnapshots(1) + self.assertCountDirectories(len(self.in_directories)) + self.assertCountRevisions(len(self.in_revisions)) + self.assertCountReleases(len(self.in_releases)) + def test_directory_cascade(self): + """Checks that sending a directory triggers sending contents""" + self.loader.load() # initialize the loader -class LoadTestContent(DummyBaseLoaderTest): - def test_load_contents(self): - """Loading contents should be ok + self.loader.maybe_load_contents(self.in_contents[0:1]) + self.loader.maybe_load_directories(self.in_directories) - """ - self.storage.content_add(self.in_contents) - self.assertCountContents(len(self.expected_contents)) - self.assertContentsOk(self.expected_contents) + self.assertCountContents(1) + self.assertCountDirectories(len(self.in_directories)) - def test_failing(self): - """Comparing wrong snapshot should fail. + def test_revision_cascade(self): + """Checks that sending a revision triggers sending contents and + directories.""" - """ - self.storage.content_add(self.in_contents) - with self.assertRaises(AssertionError): - self.assertContentsOk([]) + self.loader.load() # initialize the loader + self.loader.maybe_load_contents(self.in_contents[0:1]) + self.loader.maybe_load_directories(self.in_directories[0:1]) + self.loader.maybe_load_revisions(self.in_revisions) -class LoadTestDirectory(DummyBaseLoaderTest): - def test_send_batch_directories(self): - """Loading directories should be ok + self.assertCountContents(1) + self.assertCountDirectories(1) + self.assertCountRevisions(len(self.in_revisions)) - """ - self.storage.directory_add(self.in_directories) - self.assertCountDirectories(len(self.in_directories)) - self.assertDirectoriesOk(self.in_directories) + def test_release_cascade(self): + """Checks that sending a release triggers sending revisions, + contents, and directories.""" + self.loader.load() # initialize the loader - def test_failing(self): - """Comparing wrong snapshot should fail. + self.loader.maybe_load_contents(self.in_contents[0:1]) + self.loader.maybe_load_directories(self.in_directories[0:1]) + self.loader.maybe_load_revisions(self.in_revisions[0:1]) + self.loader.maybe_load_releases(self.in_releases) - """ - self.storage.directory_add(self.in_directories) - with self.assertRaises(AssertionError): - self.assertDirectoriesOk([]) + self.assertCountContents(1) + self.assertCountDirectories(1) + self.assertCountRevisions(1) + self.assertCountReleases(len(self.in_releases)) + + def test_snapshot_cascade(self): + """Checks that sending a snapshot triggers sending releases, + revisions, contents, and directories.""" + self.loader.load() # initialize the loader + + self.loader.maybe_load_contents(self.in_contents[0:1]) + self.loader.maybe_load_directories(self.in_directories[0:1]) + self.loader.maybe_load_revisions(self.in_revisions[0:1]) + self.loader.maybe_load_releases(self.in_releases[0:1]) + self.loader.maybe_load_snapshot(self.in_snapshot) + + self.assertCountContents(1) + self.assertCountDirectories(1) + self.assertCountRevisions(1) + self.assertCountReleases(1) + self.assertCountSnapshots(1)