Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/tests/test_loader.py
| Show All 32 Lines | def prepare_origin_visit(self, *args, **kwargs): | ||||
| self._test_prepare_origin_visit_data['origin']) | self._test_prepare_origin_visit_data['origin']) | ||||
| self.origin = origin | self.origin = origin | ||||
| self.origin_url = origin['url'] | self.origin_url = origin['url'] | ||||
| self.visit_date = datetime.datetime.utcnow() | self.visit_date = datetime.datetime.utcnow() | ||||
| self.visit_type = 'git' | self.visit_type = 'git' | ||||
| self.storage.origin_visit_add(self.origin_url, self.visit_date, | self.storage.origin_visit_add(self.origin_url, self.visit_date, | ||||
| self.visit_type) | self.visit_type) | ||||
| class DummyUnbufferedLoader(DummyLoader, UnbufferedLoader): | |||||
| """Unbuffered loader will send directly to storage new data | |||||
| """ | |||||
| def parse_config_file(self, *args, **kwargs): | def parse_config_file(self, *args, **kwargs): | ||||
| return { | return { | ||||
| 'storage': { | 'storage': { | ||||
| 'cls': 'memory', | 'cls': 'pipeline', | ||||
| 'args': { | 'steps': [ | ||||
| } | { | ||||
| 'cls': 'filter' | |||||
| }, | |||||
| { | |||||
| 'cls': 'memory' | |||||
| }, | |||||
| ] | |||||
| }, | }, | ||||
| 'send_contents': True, | |||||
| 'send_directories': True, | |||||
| 'send_revisions': True, | |||||
| 'send_releases': True, | |||||
| 'send_snapshot': True, | |||||
| 'content_packet_size': 2, | |||||
| 'content_packet_size_bytes': 8, | |||||
| 'directory_packet_size': 2, | |||||
| 'revision_packet_size': 2, | |||||
| 'release_packet_size': 2, | |||||
| 'content_size_limit': 10000, | |||||
| } | } | ||||
| class DummyUnbufferedLoader(DummyLoader, UnbufferedLoader): | |||||
| pass | |||||
| class DummyBufferedLoader(DummyLoader, BufferedLoader): | class DummyBufferedLoader(DummyLoader, BufferedLoader): | ||||
| pass | """Buffered loader will send new data when threshold is reached | ||||
| """ | |||||
| def parse_config_file(self, *args, **kwargs): | |||||
| return { | |||||
| 'storage': { | |||||
| 'cls': 'pipeline', | |||||
| 'steps': [ | |||||
| { | |||||
| 'cls': 'filter' | |||||
| }, | |||||
| { | |||||
| 'cls': 'buffer', | |||||
| 'min_batch_size': { | |||||
| 'content': 2, | |||||
| 'content_bytes': 8, | |||||
| 'directory': 2, | |||||
| 'revision': 2, | |||||
| 'release': 2, | |||||
| }, | |||||
| }, | |||||
| { | |||||
| 'cls': 'memory' | |||||
| }, | |||||
| ] | |||||
| }, | |||||
| } | |||||
| class DummyBaseLoaderTest(BaseLoaderTest): | class DummyBaseLoaderTest(BaseLoaderTest): | ||||
| def setUp(self): | def setUp(self): | ||||
| self.loader = self.loader_class(logging_class='dummyloader') | self.loader = self.loader_class(logging_class='dummyloader') | ||||
| self.loader.visit_type = 'git' | self.loader.visit_type = 'git' | ||||
| # do not call voluntarily super().setUp() | # do not call voluntarily super().setUp() | ||||
| self.storage = self.loader.storage | self.storage = self.loader.storage | ||||
| ▲ Show 20 Lines • Show All 146 Lines • ▼ Show 20 Lines | |||||
| class CoreBufferedLoaderTest(DummyBaseLoaderTest): | class CoreBufferedLoaderTest(DummyBaseLoaderTest): | ||||
| loader_class = DummyBufferedLoader | loader_class = DummyBufferedLoader | ||||
| def test_buffered_loader(self): | def test_buffered_loader(self): | ||||
| self.loader.load() # initialize the loader | self.loader.load() # initialize the loader | ||||
| self.loader.maybe_load_contents(self.in_contents[0:1]) | self.loader.send_contents(self.in_contents[0:1]) | ||||
| self.loader.maybe_load_directories(self.in_directories[0:1]) | self.loader.send_directories(self.in_directories[0:1]) | ||||
| self.loader.maybe_load_revisions(self.in_revisions[0:1]) | self.loader.send_revisions(self.in_revisions[0:1]) | ||||
| self.loader.maybe_load_releases(self.in_releases[0:1]) | self.loader.send_releases(self.in_releases[0:1]) | ||||
| self.assertCountContents(0) | self.assertCountContents(0) | ||||
| self.assertCountDirectories(0) | self.assertCountDirectories(0) | ||||
| self.assertCountRevisions(0) | self.assertCountRevisions(0) | ||||
| self.assertCountReleases(0) | self.assertCountReleases(0) | ||||
| self.loader.maybe_load_contents(self.in_contents[1:]) | self.loader.send_contents(self.in_contents[1:]) | ||||
| self.loader.maybe_load_directories(self.in_directories[1:]) | self.loader.send_directories(self.in_directories[1:]) | ||||
| self.loader.maybe_load_revisions(self.in_revisions[1:]) | self.loader.send_revisions(self.in_revisions[1:]) | ||||
| self.loader.maybe_load_releases(self.in_releases[1:]) | self.loader.send_releases(self.in_releases) | ||||
| self.assertCountContents(len(self.in_contents)) | self.assertCountContents(len(self.in_contents)) | ||||
| self.assertCountDirectories(len(self.in_directories)) | self.assertCountDirectories(len(self.in_directories)) | ||||
| self.assertCountRevisions(len(self.in_revisions)) | self.assertCountRevisions(len(self.in_revisions)) | ||||
| self.assertCountReleases(len(self.in_releases)) | self.assertCountReleases(len(self.in_releases)) | ||||
| def test_directory_cascade(self): | def test_directory_cascade(self): | ||||
| """Checks that sending a directory triggers sending contents""" | """Checks that sending a directory triggers sending contents""" | ||||
| self.loader.load() # initialize the loader | self.loader.load() # initialize the loader | ||||
| self.loader.maybe_load_contents(self.in_contents[0:1]) | self.loader.send_contents(self.in_contents[0:1]) | ||||
| self.loader.maybe_load_directories(self.in_directories) | self.loader.send_directories(self.in_directories) | ||||
| self.assertCountContents(1) | self.assertCountContents(1) | ||||
| self.assertCountDirectories(len(self.in_directories)) | self.assertCountDirectories(len(self.in_directories)) | ||||
| def test_revision_cascade(self): | def test_revision_cascade(self): | ||||
| """Checks that sending a revision triggers sending contents and | """Checks that sending a revision triggers sending contents and | ||||
| directories.""" | directories.""" | ||||
| self.loader.load() # initialize the loader | self.loader.load() # initialize the loader | ||||
| self.loader.maybe_load_contents(self.in_contents[0:1]) | self.loader.send_contents(self.in_contents[0:1]) | ||||
| self.loader.maybe_load_directories(self.in_directories[0:1]) | self.loader.send_directories(self.in_directories[0:1]) | ||||
| self.loader.maybe_load_revisions(self.in_revisions) | self.loader.send_revisions(self.in_revisions) | ||||
| self.assertCountContents(1) | self.assertCountContents(1) | ||||
| self.assertCountDirectories(1) | self.assertCountDirectories(1) | ||||
| self.assertCountRevisions(len(self.in_revisions)) | self.assertCountRevisions(len(self.in_revisions)) | ||||
| def test_release_cascade(self): | def test_release_cascade(self): | ||||
| """Checks that sending a release triggers sending revisions, | """Checks that sending a release triggers sending revisions, | ||||
| contents, and directories.""" | contents, and directories.""" | ||||
| self.loader.load() # initialize the loader | self.loader.load() # initialize the loader | ||||
| self.loader.maybe_load_contents(self.in_contents[0:1]) | self.loader.send_contents(self.in_contents[0:1]) | ||||
| self.loader.maybe_load_directories(self.in_directories[0:1]) | self.loader.send_directories(self.in_directories[0:1]) | ||||
| self.loader.maybe_load_revisions(self.in_revisions[0:1]) | self.loader.send_revisions(self.in_revisions[0:1]) | ||||
| self.loader.maybe_load_releases(self.in_releases) | self.loader.send_releases(self.in_releases) | ||||
| self.assertCountContents(1) | self.assertCountContents(1) | ||||
| self.assertCountDirectories(1) | self.assertCountDirectories(1) | ||||
| self.assertCountRevisions(1) | self.assertCountRevisions(1) | ||||
| self.assertCountReleases(len(self.in_releases)) | self.assertCountReleases(len(self.in_releases)) | ||||
| def test_snapshot_cascade(self): | def test_snapshot_cascade(self): | ||||
| """Checks that sending a snapshot triggers sending releases, | """Checks that sending a snapshot triggers sending releases, | ||||
| revisions, contents, and directories.""" | revisions, contents, and directories.""" | ||||
| self.loader.load() # initialize the loader | self.loader.load() # initialize the loader | ||||
| self.loader.maybe_load_contents(self.in_contents[0:1]) | self.loader.send_contents(self.in_contents[0:1]) | ||||
| self.loader.maybe_load_directories(self.in_directories[0:1]) | self.loader.send_directories(self.in_directories[0:1]) | ||||
| self.loader.maybe_load_revisions(self.in_revisions[0:1]) | self.loader.send_revisions(self.in_revisions[0:1]) | ||||
| self.loader.maybe_load_releases(self.in_releases[0:1]) | self.loader.send_releases(self.in_releases[0:1]) | ||||
| self.loader.maybe_load_snapshot(self.in_snapshot) | self.loader.send_snapshot(self.in_snapshot) | ||||
| self.assertCountContents(1) | self.assertCountContents(1) | ||||
| self.assertCountDirectories(1) | self.assertCountDirectories(1) | ||||
| self.assertCountRevisions(1) | self.assertCountRevisions(1) | ||||
| self.assertCountReleases(1) | self.assertCountReleases(1) | ||||
| self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
| def test_origin_metadata(self): | def test_origin_metadata(self): | ||||
| ▲ Show 20 Lines • Show All 59 Lines • Show Last 20 Lines | |||||