Page MenuHomeSoftware Heritage
Files F7123983

D3953.id13912.diff
No OneTemporary
Actions

D3953.id13912.diff
View Options

diff --git a/requirements-swh.txt b/requirements-swh.txt
--- a/requirements-swh.txt
+++ b/requirements-swh.txt
@@ -1 +1 @@
-swh.core >= 0.0.75
+swh.core[http] >= 0.0.75
diff --git a/swh/deposit/config.py b/swh/deposit/config.py
--- a/swh/deposit/config.py
+++ b/swh/deposit/config.py
@@ -1,15 +1,16 @@
-# Copyright (C) 2017-2018 The Software Heritage developers
+# Copyright (C) 2017-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import os
-import logging
-from typing import Any, Dict, Tuple
+from typing import Any, Dict
-from swh.core.config import SWHConfig
+from swh.core import config
from swh.scheduler import get_scheduler
+from swh.scheduler.interface import SchedulerInterface
+
# IRIs (Internationalized Resource identifier) sword 2.0 specified
EDIT_SE_IRI = "edit_se_iri"
@@ -49,6 +50,12 @@
}
+DEFAULT_CONFIG = {
+ "max_upload_size": 209715200,
+ "checks": True,
+}
+
+
def setup_django_for(platform=None, config_file=None):
"""Setup function for command line tools (swh.deposit.create_user) to
initialize the needed db access.
@@ -81,30 +88,14 @@
django.setup()
-class APIConfig(SWHConfig):
- """Mixin intended to enrich views with SWH configuration.
+class APIConfig:
+ """API Configuration centralized class. This loads explicitly the configuration file out
+ of the SWH_CONFIG_FILENAME environment variable.
"""
- CONFIG_BASE_FILENAME = "deposit/server"
-
- DEFAULT_CONFIG = {
- "max_upload_size": ("int", 209715200),
- "checks": ("bool", True),
- "scheduler": (
- "dict",
- {"cls": "remote", "args": {"url": "http://localhost:5008/"}},
- ),
- }
-
- ADDITIONAL_CONFIG = {} # type: Dict[str, Tuple[str, Any]]
-
- def __init__(self, **config):
- super().__init__()
- self.config = self.parse_config_file(
- additional_configs=[self.ADDITIONAL_CONFIG]
- )
- self.config.update(config)
- self.log = logging.getLogger("swh.deposit")
- if self.config.get("scheduler"):
- self.scheduler = get_scheduler(**self.config["scheduler"])
+ def __init__(self):
+ config_file = os.environ["SWH_CONFIG_FILENAME"]
+ conf = config.read_raw_config(config.config_basepath(config_file))
+ self.config: Dict[str, Any] = config.merge_configs(DEFAULT_CONFIG, conf)
+ self.scheduler: SchedulerInterface = get_scheduler(**self.config["scheduler"])
diff --git a/swh/deposit/tests/api/test_deposit_schedule.py b/swh/deposit/tests/api/test_deposit_schedule.py
--- a/swh/deposit/tests/api/test_deposit_schedule.py
+++ b/swh/deposit/tests/api/test_deposit_schedule.py
@@ -3,9 +3,10 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import copy
import datetime
+
from io import BytesIO
-from typing import Dict
from django.urls import reverse
import pytest
@@ -18,20 +19,13 @@
from swh.deposit.parsers import parse_xml
-from ..conftest import TEST_CONFIG
-
-
-TEST_CONFIG_WITH_CHECKS: Dict[str, object] = {
- **TEST_CONFIG,
- "checks": True,
-}
-
-
@pytest.fixture()
-def deposit_config():
+def deposit_config(deposit_config):
"""Overrides the `deposit_config` fixture define in swh/deposit/tests/conftest.py
to re-enable the checks."""
- return TEST_CONFIG_WITH_CHECKS
+ config_d = copy.deepcopy(deposit_config)
+ config_d["checks"] = True
+ return config_d
def now() -> datetime.datetime:
diff --git a/swh/deposit/tests/conftest.py b/swh/deposit/tests/conftest.py
--- a/swh/deposit/tests/conftest.py
+++ b/swh/deposit/tests/conftest.py
@@ -8,6 +8,8 @@
import pytest
import psycopg2
+import yaml
+
from django.urls import reverse
from django.test.utils import setup_databases # type: ignore
@@ -23,7 +25,6 @@
from swh.model.identifiers import DIRECTORY, swhid, REVISION, SNAPSHOT
from swh.deposit.config import setup_django_for
from swh.deposit.parsers import parse_xml
-from swh.deposit.config import APIConfig
from swh.deposit.config import (
COL_IRI,
EDIT_SE_IRI,
@@ -47,52 +48,49 @@
}
-TEST_CONFIG = {
- "max_upload_size": 500,
- "extraction_dir": "/tmp/swh-deposit/test/extraction-dir",
- "checks": False,
- "provider": {
- "provider_name": "",
- "provider_type": "deposit_client",
- "provider_url": "",
- "metadata": {},
- },
- "tool": {
- "name": "swh-deposit",
- "version": "0.0.1",
- "configuration": {"sword_version": "2"},
- },
-}
-
-
def pytest_configure():
setup_django_for("testing")
@pytest.fixture()
-def deposit_config():
- return TEST_CONFIG
+def deposit_config(swh_scheduler_config):
+ return {
+ "max_upload_size": 500,
+ "extraction_dir": "/tmp/swh-deposit/test/extraction-dir",
+ "checks": False,
+ "provider": {
+ "provider_name": "",
+ "provider_type": "deposit_client",
+ "provider_url": "",
+ "metadata": {},
+ },
+ "tool": {
+ "name": "swh-deposit",
+ "version": "0.0.1",
+ "configuration": {"sword_version": "2"},
+ },
+ "scheduler": {"cls": "local", "args": swh_scheduler_config,},
+ }
-@pytest.fixture(autouse=True)
-def deposit_autoconfig(monkeypatch, deposit_config, swh_scheduler_config):
- """Enforce config for deposit classes inherited from APIConfig."""
+@pytest.fixture()
+def deposit_config_path(tmp_path, monkeypatch, deposit_config):
+ conf_path = os.path.join(tmp_path, "deposit.yml")
+ with open(conf_path, "w") as f:
+ f.write(yaml.dump(deposit_config))
+ monkeypatch.setenv("SWH_CONFIG_FILENAME", conf_path)
+ return conf_path
- def mock_parse_config(*args, **kw):
- config = deposit_config.copy()
- config["scheduler"] = {
- "cls": "local",
- "args": swh_scheduler_config,
- }
- return config
- monkeypatch.setattr(APIConfig, "parse_config_file", mock_parse_config)
+@pytest.fixture(autouse=True)
+def deposit_autoconfig(deposit_config_path, swh_scheduler_config):
+ """Enforce config for deposit classes inherited from APIConfig."""
scheduler = get_scheduler("local", swh_scheduler_config)
task_type = {
"type": "load-deposit",
"backend_name": "swh.loader.packages.deposit.tasks.LoadDeposit",
- "description": "why does this have not-null constraint?",
+ "description": "Load deposit task",
}
scheduler.create_task_type(task_type)

File Metadata

Mime Type
text/plain
Expires
Dec 20 2024, 7:50 AM (11 w, 3 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3226341

Event Timeline

About · Developer information · Legal