diff --git a/swh/loader/core/loader.py b/swh/loader/core/loader.py --- a/swh/loader/core/loader.py +++ b/swh/loader/core/loader.py @@ -10,7 +10,7 @@ import os from typing import Any, Dict, Iterable, Optional, Tuple -from swh.core import config +from swh.core.config import load_from_envvar from swh.model.model import ( BaseContent, Content, @@ -27,13 +27,20 @@ from swh.storage import get_storage from swh.storage.utils import now +DEFAULT_CONFIG: Dict[str, Any] = { + "max_content_size": 100 * 1024 * 1024, + "save_data": False, + "save_data_path": "", +} -class BaseLoader(config.SWHConfig, metaclass=ABCMeta): + +class BaseLoader(metaclass=ABCMeta): """Mixin base class for loader. To use this class, you must: - inherit from this class + - and implement the @abstractmethod methods: - :func:`prepare`: First step executed by the loader to prepare some @@ -58,30 +65,14 @@ You can take a look at some example classes: - - :class:`BaseSvnLoader` + - :class:`SvnLoader` """ - CONFIG_BASE_FILENAME = None # type: Optional[str] - - DEFAULT_CONFIG = { - "storage": ("dict", {"cls": "remote", "url": "http://localhost:5002/",}), - "max_content_size": ("int", 100 * 1024 * 1024), - "save_data": ("bool", False), - "save_data_path": ("str", ""), - } # type: Dict[str, Tuple[str, Any]] - - ADDITIONAL_CONFIG = {} # type: Dict[str, Tuple[str, Any]] - def __init__( - self, logging_class: Optional[str] = None, config: Dict[str, Any] = {} + self, logging_class: Optional[str] = None, ): - if config: - self.config = config - else: - self.config = self.parse_config_file( - additional_configs=[self.ADDITIONAL_CONFIG] - ) + self.config = load_from_envvar(DEFAULT_CONFIG) self.storage = get_storage(**self.config["storage"]) diff --git a/swh/loader/core/tests/test_loader.py b/swh/loader/core/tests/test_loader.py --- a/swh/loader/core/tests/test_loader.py +++ b/swh/loader/core/tests/test_loader.py @@ -15,6 +15,9 @@ class DummyLoader: + """Base Loader to overload and simplify the base class (technical: to avoid repetition + in other *Loader classes)""" + def cleanup(self): pass @@ -40,19 +43,10 @@ class DummyDVCSLoader(DummyLoader, DVCSLoader): - """Unbuffered loader will send directly to storage new data + """DVCS Loader that does nothing in regards to DAG objects. """ - def parse_config_file(self, *args, **kwargs): - return { - "max_content_size": 100 * 1024 * 1024, - "storage": { - "cls": "pipeline", - "steps": [{"cls": "retry",}, {"cls": "filter",}, {"cls": "memory",},], - }, - } - def get_contents(self): return [] @@ -77,47 +71,23 @@ """ - def parse_config_file(self, *args, **kwargs): - return { - "max_content_size": 100 * 1024 * 1024, - "storage": { - "cls": "pipeline", - "steps": [ - {"cls": "retry",}, - {"cls": "filter",}, - { - "cls": "buffer", - "min_batch_size": { - "content": 2, - "content_bytes": 8, - "directory": 2, - "revision": 2, - "release": 2, - }, - }, - {"cls": "memory",}, - ], - }, - } - def store_data(self): pass -def test_base_loader(): +def test_base_loader(swh_config): loader = DummyBaseLoader() result = loader.load() - assert result == {"status": "eventful"} -def test_dvcs_loader(): +def test_dvcs_loader(swh_config): loader = DummyDVCSLoader() result = loader.load() assert result == {"status": "eventful"} -def test_loader_logger_default_name(): +def test_loader_logger_default_name(swh_config): loader = DummyBaseLoader() assert isinstance(loader.log, logging.Logger) assert loader.log.name == "swh.loader.core.tests.test_loader.DummyBaseLoader" @@ -127,13 +97,13 @@ assert loader.log.name == "swh.loader.core.tests.test_loader.DummyDVCSLoader" -def test_loader_logger_with_name(): +def test_loader_logger_with_name(swh_config): loader = DummyBaseLoader("some.logger.name") assert isinstance(loader.log, logging.Logger) assert loader.log.name == "some.logger.name" -def test_loader_save_data_path(tmp_path): +def test_loader_save_data_path(swh_config, tmp_path): loader = DummyBaseLoader("some.logger.name.1") url = "http://bitbucket.org/something" loader.origin = Origin(url=url) @@ -179,7 +149,7 @@ raise RuntimeError("Failed to get contents!") -def test_dvcs_loader_exc_partial_visit(caplog): +def test_dvcs_loader_exc_partial_visit(swh_config, caplog): logger_name = "dvcsloaderexc" caplog.set_level(logging.ERROR, logger=logger_name) @@ -210,7 +180,7 @@ self.storage = BrokenStorageProxy(self.storage) -def test_dvcs_loader_storage_exc_partial_visit(caplog): +def test_dvcs_loader_storage_exc_partial_visit(swh_config, caplog): logger_name = "dvcsloaderexc" caplog.set_level(logging.ERROR, logger=logger_name)