diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,5 +1,5 @@ swh.core >= 0.0.7 -swh.loader.core >= 0.0.37 +swh.loader.core >= 0.0.67 swh.model >= 0.0.27 swh.scheduler >= 0.0.39 swh.storage >= 0.0.108 diff --git a/swh/loader/git/from_disk.py b/swh/loader/git/from_disk.py --- a/swh/loader/git/from_disk.py +++ b/swh/loader/git/from_disk.py @@ -147,7 +147,7 @@ def get_contents(self): """Get the contents that need to be loaded""" - max_content_size = self.config['content_size_limit'] + max_content_size = self.config['max_content_size'] missing_contents = set(self.storage.content_missing( self.get_content_ids(), 'sha1_git')) diff --git a/swh/loader/git/loader.py b/swh/loader/git/loader.py --- a/swh/loader/git/loader.py +++ b/swh/loader/git/loader.py @@ -379,7 +379,7 @@ def get_contents(self): """Format the blobs from the git repository as swh contents""" - max_content_size = self.config['content_size_limit'] + max_content_size = self.config['max_content_size'] missing_contents = set(self.storage.content_missing( self.get_content_ids(), 'sha1_git')) diff --git a/swh/loader/git/tests/__init__.py b/swh/loader/git/tests/__init__.py --- a/swh/loader/git/tests/__init__.py +++ b/swh/loader/git/tests/__init__.py @@ -1,21 +1,31 @@ +# Copyright (C) 2018-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + TEST_LOADER_CONFIG = { 'storage': { - 'cls': 'memory', - 'args': { - } + 'cls': 'pipeline', + 'steps': [ + { + 'cls': 'filter' + }, + { + 'cls': 'buffer', + 'min_batch_size': { + 'content': 10, + 'content_bytes': 100 * 1024 * 1024, + 'directory': 10, + 'revision': 10, + 'release': 10, + }, + }, + { + 'cls': 'memory' + }, + ] }, - 'send_contents': True, - 'send_directories': True, - 'send_revisions': True, - 'send_releases': True, - 'send_snapshot': True, - - 'content_size_limit': 100 * 1024 * 1024, - 'content_packet_size': 10, - 'content_packet_size_bytes': 100 * 1024 * 1024, - 'directory_packet_size': 10, - 'revision_packet_size': 10, - 'release_packet_size': 10, - + 'max_content_size': 100 * 1024 * 1024, + 'pack_size_bytes': 4 * 1024 * 1024 * 1024, 'save_data': False, } diff --git a/swh/loader/git/tests/conftest.py b/swh/loader/git/tests/conftest.py --- a/swh/loader/git/tests/conftest.py +++ b/swh/loader/git/tests/conftest.py @@ -1,3 +1,8 @@ +# Copyright (C) 2018-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + import pytest from swh.scheduler.tests.conftest import * # noqa diff --git a/swh/loader/git/tests/test_loader.py b/swh/loader/git/tests/test_loader.py --- a/swh/loader/git/tests/test_loader.py +++ b/swh/loader/git/tests/test_loader.py @@ -3,16 +3,17 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information - from swh.loader.git.loader import GitLoader from swh.loader.git.tests.test_from_disk import DirGitLoaderTest +from . import TEST_LOADER_CONFIG + class GitLoaderTest(GitLoader): def parse_config_file(self, *args, **kwargs): return { **super().parse_config_file(*args, **kwargs), - 'storage': {'cls': 'memory', 'args': {}} + **TEST_LOADER_CONFIG }