diff --git a/swh/loader/git/from_disk.py b/swh/loader/git/from_disk.py --- a/swh/loader/git/from_disk.py +++ b/swh/loader/git/from_disk.py @@ -33,8 +33,6 @@ """ - CONFIG_BASE_FILENAME = "loader/git-disk" - visit_type = "git" def __init__(self, url, visit_date=None, directory=None, config=None): diff --git a/swh/loader/git/loader.py b/swh/loader/git/loader.py --- a/swh/loader/git/loader.py +++ b/swh/loader/git/loader.py @@ -17,6 +17,7 @@ from dulwich.object_store import ObjectStoreGraphWalker from dulwich.pack import PackData, PackInflater +from swh.core.config import merge_configs from swh.loader.core.loader import DVCSLoader from swh.model import hashutil from swh.model.model import ( @@ -95,14 +96,13 @@ pack_size: int -class GitLoader(DVCSLoader): - """A bulk loader for a git repository""" +DEFAULT_CONFIG: Dict[str, Any] = { + "pack_size_bytes": 4 * 1024 * 1024 * 1024, +} - CONFIG_BASE_FILENAME = "loader/git" - ADDITIONAL_CONFIG = { - "pack_size_bytes": ("int", 4 * 1024 * 1024 * 1024), - } +class GitLoader(DVCSLoader): + """A bulk loader for a git repository""" visit_type = "git" @@ -112,7 +112,6 @@ base_url: Optional[str] = None, ignore_history: bool = False, repo_representation: Type[RepoRepresentation] = RepoRepresentation, - config: Optional[Dict[str, Any]] = None, ): """Initialize the bulk updater. @@ -122,10 +121,8 @@ data. """ - if config is None: - config = {} - - super().__init__(logging_class="swh.loader.git.BulkLoader", config=config) + super().__init__(logging_class="swh.loader.git.BulkLoader") + self.config = merge_configs(DEFAULT_CONFIG, self.config) self.origin_url = url self.base_url = base_url self.ignore_history = ignore_history diff --git a/swh/loader/git/tests/test_tasks.py b/swh/loader/git/tests/test_tasks.py --- a/swh/loader/git/tests/test_tasks.py +++ b/swh/loader/git/tests/test_tasks.py @@ -1,10 +1,12 @@ -# Copyright (C) 2018-2019 The Software Heritage developers +# Copyright (C) 2018-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -def test_git_loader(mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker): +def test_git_loader( + mocker, swh_config, swh_scheduler_celery_app, swh_scheduler_celery_worker +): mock_loader = mocker.patch("swh.loader.git.loader.GitLoader.load") mock_loader.return_value = {"status": "eventful"} @@ -20,7 +22,7 @@ def test_git_loader_from_disk( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker + mocker, swh_config, swh_scheduler_celery_app, swh_scheduler_celery_worker ): mock_loader = mocker.patch("swh.loader.git.from_disk.GitLoaderFromDisk.load") mock_loader.return_value = {"status": "uneventful"} @@ -42,7 +44,7 @@ def test_git_loader_from_archive( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker + mocker, swh_config, swh_scheduler_celery_app, swh_scheduler_celery_worker ): mock_loader = mocker.patch("swh.loader.git.from_disk.GitLoaderFromArchive.load")