Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/tests/test_loader.py
| # Copyright (C) 2018 The Software Heritage developers | # Copyright (C) 2018 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import datetime | import datetime | ||||
| from swh.model.hashutil import hash_to_bytes | from swh.model.hashutil import hash_to_bytes | ||||
| from swh.loader.core.loader import SWHLoader | from swh.loader.core.loader import BufferedLoader, UnbufferedLoader | ||||
| from . import BaseLoaderTest | from . import BaseLoaderTest | ||||
| class DummyLoader(SWHLoader): | class DummyLoader: | ||||
| def cleanup(self): | def cleanup(self): | ||||
| pass | pass | ||||
| def prepare(self): | def prepare(self): | ||||
| pass | pass | ||||
| def fetch_data(self): | def fetch_data(self): | ||||
| pass | pass | ||||
| Show All 29 Lines | def parse_config_file(self, *args, **kwargs): | ||||
| 'directory_packet_size': 2, | 'directory_packet_size': 2, | ||||
| 'revision_packet_size': 2, | 'revision_packet_size': 2, | ||||
| 'release_packet_size': 2, | 'release_packet_size': 2, | ||||
| 'content_size_limit': 10000, | 'content_size_limit': 10000, | ||||
| } | } | ||||
| class DummyUnbufferedLoader(DummyLoader, UnbufferedLoader): | |||||
| pass | |||||
| class DummyBufferedLoader(DummyLoader, BufferedLoader): | |||||
| pass | |||||
| class DummyBaseLoaderTest(BaseLoaderTest): | class DummyBaseLoaderTest(BaseLoaderTest): | ||||
| def setUp(self): | def setUp(self): | ||||
| self.loader = DummyLoader(logging_class='dummyloader') | self.loader = self.loader_class(logging_class='dummyloader') | ||||
| # do not call voluntarily super().setUp() | # do not call voluntarily super().setUp() | ||||
| self.storage = self.loader.storage | self.storage = self.loader.storage | ||||
| contents = [ | contents = [ | ||||
| { | { | ||||
| 'id': '34973274ccef6ab4dfaaf86599792fa9c3fe4689', | 'id': '34973274ccef6ab4dfaaf86599792fa9c3fe4689', | ||||
| 'sha1': '34973274ccef6ab4dfaaf86599792fa9c3fe4689', | 'sha1': '34973274ccef6ab4dfaaf86599792fa9c3fe4689', | ||||
| 'sha1_git': b'bar1', | 'sha1_git': b'bar1', | ||||
| 'sha256': b'baz1', | 'sha256': b'baz1', | ||||
| ▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | def setUp(self): | ||||
| 'origin': self.in_origins[0], | 'origin': self.in_origins[0], | ||||
| } | } | ||||
| def tearDown(self): | def tearDown(self): | ||||
| # do not call voluntarily super().tearDown() | # do not call voluntarily super().tearDown() | ||||
| pass | pass | ||||
| class CoreLoaderTest(DummyBaseLoaderTest): | class CoreUnbufferedLoaderTest(DummyBaseLoaderTest): | ||||
| def test_stateful_loader(self): | loader_class = DummyUnbufferedLoader | ||||
| """Stateful loader accumulates in place the sent data | |||||
| def test_unbuffered_loader(self): | |||||
| self.loader.load() # initialize the loader | |||||
| Note: Those behaviors should be somehow merged but that's | self.loader.send_contents(self.in_contents[0:1]) | ||||
| another story. | self.loader.send_directories(self.in_directories[0:1]) | ||||
| self.loader.send_revisions(self.in_revisions[0:1]) | |||||
| self.loader.send_releases(self.in_releases[0:1]) | |||||
| self.assertCountContents(1) | |||||
| self.assertCountDirectories(1) | |||||
| self.assertCountRevisions(1) | |||||
| self.assertCountReleases(1) | |||||
| self.loader.send_contents(self.in_contents[1:]) | |||||
| self.loader.send_directories(self.in_directories[1:]) | |||||
| self.loader.send_revisions(self.in_revisions[1:]) | |||||
| self.loader.send_releases(self.in_releases[1:]) | |||||
| self.assertCountContents(len(self.in_contents)) | |||||
| self.assertCountDirectories(len(self.in_directories)) | |||||
| self.assertCountRevisions(len(self.in_revisions)) | |||||
| self.assertCountReleases(len(self.in_releases)) | |||||
| """ | |||||
| class CoreBufferedLoaderTest(DummyBaseLoaderTest): | |||||
| loader_class = DummyBufferedLoader | |||||
| def test_buffered_loader(self): | |||||
| self.loader.load() # initialize the loader | self.loader.load() # initialize the loader | ||||
| self.loader.maybe_load_contents(self.in_contents[0:1]) | self.loader.maybe_load_contents(self.in_contents[0:1]) | ||||
| self.loader.maybe_load_directories(self.in_directories[0:1]) | self.loader.maybe_load_directories(self.in_directories[0:1]) | ||||
| self.loader.maybe_load_revisions(self.in_revisions[0:1]) | self.loader.maybe_load_revisions(self.in_revisions[0:1]) | ||||
| self.loader.maybe_load_releases(self.in_releases[0:1]) | self.loader.maybe_load_releases(self.in_releases[0:1]) | ||||
| self.assertCountContents(0) | self.assertCountContents(0) | ||||
| self.assertCountDirectories(0) | self.assertCountDirectories(0) | ||||
| self.assertCountRevisions(0) | self.assertCountRevisions(0) | ||||
| self.assertCountReleases(0) | self.assertCountReleases(0) | ||||
| self.loader.maybe_load_contents(self.in_contents[1:]) | self.loader.maybe_load_contents(self.in_contents[1:]) | ||||
| self.loader.maybe_load_directories(self.in_directories[1:]) | self.loader.maybe_load_directories(self.in_directories[1:]) | ||||
| self.loader.maybe_load_revisions(self.in_revisions[1:]) | self.loader.maybe_load_revisions(self.in_revisions[1:]) | ||||
| self.loader.maybe_load_releases(self.in_releases[1:]) | self.loader.maybe_load_releases(self.in_releases[1:]) | ||||
| self.assertCountContents(len(self.in_contents)) | self.assertCountContents(len(self.in_contents)) | ||||
| self.assertCountDirectories(len(self.in_directories)) | self.assertCountDirectories(len(self.in_directories)) | ||||
| self.assertCountRevisions(len(self.in_revisions)) | self.assertCountRevisions(len(self.in_revisions)) | ||||
| self.assertCountReleases(len(self.in_releases)) | self.assertCountReleases(len(self.in_releases)) | ||||
| def test_stateless_loader(self): | |||||
| """Stateless loader accumulates in place the sent data as well | |||||
| Note: Those behaviors should be somehow merged but that's | |||||
| another story. | |||||
| """ | |||||
| self.loader.load() # initialize the loader | |||||
| self.loader.send_contents(self.in_contents[0:1]) | |||||
| self.loader.send_directories(self.in_directories[0:1]) | |||||
| self.loader.send_revisions(self.in_revisions[0:1]) | |||||
| self.loader.send_releases(self.in_releases[0:1]) | |||||
| self.assertCountContents(1) | |||||
| self.assertCountDirectories(1) | |||||
| self.assertCountRevisions(1) | |||||
| self.assertCountReleases(1) | |||||
| self.loader.send_contents(self.in_contents[1:]) | |||||
| self.loader.send_directories(self.in_directories[1:]) | |||||
| self.loader.send_revisions(self.in_revisions[1:]) | |||||
| self.loader.send_releases(self.in_releases[1:]) | |||||
| self.assertCountContents(len(self.in_contents)) | |||||
| self.assertCountDirectories(len(self.in_directories)) | |||||
| self.assertCountRevisions(len(self.in_revisions)) | |||||
| self.assertCountReleases(len(self.in_releases)) | |||||
| def test_directory_cascade(self): | def test_directory_cascade(self): | ||||
| """Checks that sending a directory triggers sending contents""" | """Checks that sending a directory triggers sending contents""" | ||||
| self.loader.load() # initialize the loader | self.loader.load() # initialize the loader | ||||
| self.loader.maybe_load_contents(self.in_contents[0:1]) | self.loader.maybe_load_contents(self.in_contents[0:1]) | ||||
| self.loader.maybe_load_directories(self.in_directories) | self.loader.maybe_load_directories(self.in_directories) | ||||
| self.assertCountContents(1) | self.assertCountContents(1) | ||||
| ▲ Show 20 Lines • Show All 47 Lines • Show Last 20 Lines | |||||