diff --git a/swh/loader/core/loader.py b/swh/loader/core/loader.py --- a/swh/loader/core/loader.py +++ b/swh/loader/core/loader.py @@ -10,7 +10,7 @@ import os from typing import Any, Dict, Iterable, Optional, Tuple -from swh.core import config +from swh.core.config import load_from_envvar from swh.model.model import ( BaseContent, Content, @@ -27,13 +27,20 @@ from swh.storage import get_storage from swh.storage.utils import now +DEFAULT_CONFIG: Dict[str, Any] = { + "max_content_size": 100 * 1024 * 1024, + "save_data": False, + "save_data_path": "", +} -class BaseLoader(config.SWHConfig, metaclass=ABCMeta): + +class BaseLoader(metaclass=ABCMeta): """Mixin base class for loader. To use this class, you must: - inherit from this class + - and implement the @abstractmethod methods: - :func:`prepare`: First step executed by the loader to prepare some @@ -58,30 +65,17 @@ You can take a look at some example classes: - - :class:`BaseSvnLoader` + - :class:`SvnLoader` """ - CONFIG_BASE_FILENAME = None # type: Optional[str] - - DEFAULT_CONFIG = { - "storage": ("dict", {"cls": "remote", "url": "http://localhost:5002/",}), - "max_content_size": ("int", 100 * 1024 * 1024), - "save_data": ("bool", False), - "save_data_path": ("str", ""), - } # type: Dict[str, Tuple[str, Any]] - - ADDITIONAL_CONFIG = {} # type: Dict[str, Tuple[str, Any]] - def __init__( self, logging_class: Optional[str] = None, config: Dict[str, Any] = {} ): if config: self.config = config else: - self.config = self.parse_config_file( - additional_configs=[self.ADDITIONAL_CONFIG] - ) + self.config = load_from_envvar(DEFAULT_CONFIG) self.storage = get_storage(**self.config["storage"]) diff --git a/swh/loader/core/tests/test_loader.py b/swh/loader/core/tests/test_loader.py --- a/swh/loader/core/tests/test_loader.py +++ b/swh/loader/core/tests/test_loader.py @@ -6,6 +6,9 @@ import datetime import hashlib import logging +from typing import Any, Dict + +import pytest from swh.loader.core.loader import BaseLoader, DVCSLoader from swh.loader.tests import assert_last_visit_matches @@ -14,7 +17,29 @@ ORIGIN = Origin(url="some-url") +@pytest.fixture +def swh_loader_config(swh_storage_postgresql) -> Dict[str, Any]: + return { + "storage": { + "cls": "pipeline", + "steps": [ + {"cls": "retry"}, + {"cls": "filter"}, + {"cls": "buffer"}, + { + "cls": "local", + "db": swh_storage_postgresql.dsn, + "objstorage": {"cls": "memory", "args": {}}, + }, + ], + }, + } + + class DummyLoader: + """Base Loader to overload and simplify the base class (technical: to avoid repetition + in other *Loader classes)""" + def cleanup(self): pass @@ -40,19 +65,10 @@ class DummyDVCSLoader(DummyLoader, DVCSLoader): - """Unbuffered loader will send directly to storage new data + """DVCS Loader that does nothing in regards to DAG objects. """ - def parse_config_file(self, *args, **kwargs): - return { - "max_content_size": 100 * 1024 * 1024, - "storage": { - "cls": "pipeline", - "steps": [{"cls": "retry",}, {"cls": "filter",}, {"cls": "memory",},], - }, - } - def get_contents(self): return [] @@ -77,47 +93,23 @@ """ - def parse_config_file(self, *args, **kwargs): - return { - "max_content_size": 100 * 1024 * 1024, - "storage": { - "cls": "pipeline", - "steps": [ - {"cls": "retry",}, - {"cls": "filter",}, - { - "cls": "buffer", - "min_batch_size": { - "content": 2, - "content_bytes": 8, - "directory": 2, - "revision": 2, - "release": 2, - }, - }, - {"cls": "memory",}, - ], - }, - } - def store_data(self): pass -def test_base_loader(): +def test_base_loader(swh_config): loader = DummyBaseLoader() result = loader.load() - assert result == {"status": "eventful"} -def test_dvcs_loader(): +def test_dvcs_loader(swh_config): loader = DummyDVCSLoader() result = loader.load() assert result == {"status": "eventful"} -def test_loader_logger_default_name(): +def test_loader_logger_default_name(swh_config): loader = DummyBaseLoader() assert isinstance(loader.log, logging.Logger) assert loader.log.name == "swh.loader.core.tests.test_loader.DummyBaseLoader" @@ -127,13 +119,13 @@ assert loader.log.name == "swh.loader.core.tests.test_loader.DummyDVCSLoader" -def test_loader_logger_with_name(): +def test_loader_logger_with_name(swh_config): loader = DummyBaseLoader("some.logger.name") assert isinstance(loader.log, logging.Logger) assert loader.log.name == "some.logger.name" -def test_loader_save_data_path(tmp_path): +def test_loader_save_data_path(swh_config, tmp_path): loader = DummyBaseLoader("some.logger.name.1") url = "http://bitbucket.org/something" loader.origin = Origin(url=url) @@ -179,7 +171,7 @@ raise RuntimeError("Failed to get contents!") -def test_dvcs_loader_exc_partial_visit(caplog): +def test_dvcs_loader_exc_partial_visit(swh_config, caplog): logger_name = "dvcsloaderexc" caplog.set_level(logging.ERROR, logger=logger_name) @@ -210,7 +202,7 @@ self.storage = BrokenStorageProxy(self.storage) -def test_dvcs_loader_storage_exc_partial_visit(caplog): +def test_dvcs_loader_storage_exc_partial_visit(swh_config, caplog): logger_name = "dvcsloaderexc" caplog.set_level(logging.ERROR, logger=logger_name)