Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9341783
D4124.id14527.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
6 KB
Subscribers
None
D4124.id14527.diff
View Options
diff --git a/swh/loader/core/loader.py b/swh/loader/core/loader.py
--- a/swh/loader/core/loader.py
+++ b/swh/loader/core/loader.py
@@ -10,7 +10,7 @@
import os
from typing import Any, Dict, Iterable, Optional, Tuple
-from swh.core import config
+from swh.core.config import load_from_envvar
from swh.model.model import (
BaseContent,
Content,
@@ -27,13 +27,20 @@
from swh.storage import get_storage
from swh.storage.utils import now
+DEFAULT_CONFIG: Dict[str, Any] = {
+ "max_content_size": 100 * 1024 * 1024,
+ "save_data": False,
+ "save_data_path": "",
+}
-class BaseLoader(config.SWHConfig, metaclass=ABCMeta):
+
+class BaseLoader(metaclass=ABCMeta):
"""Mixin base class for loader.
To use this class, you must:
- inherit from this class
+
- and implement the @abstractmethod methods:
- :func:`prepare`: First step executed by the loader to prepare some
@@ -58,30 +65,17 @@
You can take a look at some example classes:
- - :class:`BaseSvnLoader`
+ - :class:`SvnLoader`
"""
- CONFIG_BASE_FILENAME = None # type: Optional[str]
-
- DEFAULT_CONFIG = {
- "storage": ("dict", {"cls": "remote", "url": "http://localhost:5002/",}),
- "max_content_size": ("int", 100 * 1024 * 1024),
- "save_data": ("bool", False),
- "save_data_path": ("str", ""),
- } # type: Dict[str, Tuple[str, Any]]
-
- ADDITIONAL_CONFIG = {} # type: Dict[str, Tuple[str, Any]]
-
def __init__(
self, logging_class: Optional[str] = None, config: Dict[str, Any] = {}
):
if config:
self.config = config
else:
- self.config = self.parse_config_file(
- additional_configs=[self.ADDITIONAL_CONFIG]
- )
+ self.config = load_from_envvar(DEFAULT_CONFIG)
self.storage = get_storage(**self.config["storage"])
diff --git a/swh/loader/core/tests/test_loader.py b/swh/loader/core/tests/test_loader.py
--- a/swh/loader/core/tests/test_loader.py
+++ b/swh/loader/core/tests/test_loader.py
@@ -6,6 +6,9 @@
import datetime
import hashlib
import logging
+from typing import Any, Dict
+
+import pytest
from swh.loader.core.loader import BaseLoader, DVCSLoader
from swh.loader.tests import assert_last_visit_matches
@@ -14,7 +17,29 @@
ORIGIN = Origin(url="some-url")
+@pytest.fixture
+def swh_loader_config(swh_storage_postgresql) -> Dict[str, Any]:
+ return {
+ "storage": {
+ "cls": "pipeline",
+ "steps": [
+ {"cls": "retry"},
+ {"cls": "filter"},
+ {"cls": "buffer"},
+ {
+ "cls": "local",
+ "db": swh_storage_postgresql.dsn,
+ "objstorage": {"cls": "memory", "args": {}},
+ },
+ ],
+ },
+ }
+
+
class DummyLoader:
+ """Base Loader to overload and simplify the base class (technical: to avoid repetition
+ in other *Loader classes)"""
+
def cleanup(self):
pass
@@ -40,19 +65,10 @@
class DummyDVCSLoader(DummyLoader, DVCSLoader):
- """Unbuffered loader will send directly to storage new data
+ """DVCS Loader that does nothing in regards to DAG objects.
"""
- def parse_config_file(self, *args, **kwargs):
- return {
- "max_content_size": 100 * 1024 * 1024,
- "storage": {
- "cls": "pipeline",
- "steps": [{"cls": "retry",}, {"cls": "filter",}, {"cls": "memory",},],
- },
- }
-
def get_contents(self):
return []
@@ -77,47 +93,23 @@
"""
- def parse_config_file(self, *args, **kwargs):
- return {
- "max_content_size": 100 * 1024 * 1024,
- "storage": {
- "cls": "pipeline",
- "steps": [
- {"cls": "retry",},
- {"cls": "filter",},
- {
- "cls": "buffer",
- "min_batch_size": {
- "content": 2,
- "content_bytes": 8,
- "directory": 2,
- "revision": 2,
- "release": 2,
- },
- },
- {"cls": "memory",},
- ],
- },
- }
-
def store_data(self):
pass
-def test_base_loader():
+def test_base_loader(swh_config):
loader = DummyBaseLoader()
result = loader.load()
-
assert result == {"status": "eventful"}
-def test_dvcs_loader():
+def test_dvcs_loader(swh_config):
loader = DummyDVCSLoader()
result = loader.load()
assert result == {"status": "eventful"}
-def test_loader_logger_default_name():
+def test_loader_logger_default_name(swh_config):
loader = DummyBaseLoader()
assert isinstance(loader.log, logging.Logger)
assert loader.log.name == "swh.loader.core.tests.test_loader.DummyBaseLoader"
@@ -127,13 +119,13 @@
assert loader.log.name == "swh.loader.core.tests.test_loader.DummyDVCSLoader"
-def test_loader_logger_with_name():
+def test_loader_logger_with_name(swh_config):
loader = DummyBaseLoader("some.logger.name")
assert isinstance(loader.log, logging.Logger)
assert loader.log.name == "some.logger.name"
-def test_loader_save_data_path(tmp_path):
+def test_loader_save_data_path(swh_config, tmp_path):
loader = DummyBaseLoader("some.logger.name.1")
url = "http://bitbucket.org/something"
loader.origin = Origin(url=url)
@@ -179,7 +171,7 @@
raise RuntimeError("Failed to get contents!")
-def test_dvcs_loader_exc_partial_visit(caplog):
+def test_dvcs_loader_exc_partial_visit(swh_config, caplog):
logger_name = "dvcsloaderexc"
caplog.set_level(logging.ERROR, logger=logger_name)
@@ -210,7 +202,7 @@
self.storage = BrokenStorageProxy(self.storage)
-def test_dvcs_loader_storage_exc_partial_visit(caplog):
+def test_dvcs_loader_storage_exc_partial_visit(swh_config, caplog):
logger_name = "dvcsloaderexc"
caplog.set_level(logging.ERROR, logger=logger_name)
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Thu, Jul 3, 12:18 PM (2 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3234637
Attached To
D4124: core.loader: Migrate away from SWHConfig mixin
Event Timeline
Log In to Comment