diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1 +1 @@ -swh.core >= 0.0.75 +swh.core[http] >= 0.0.75 diff --git a/swh/deposit/api/private/deposit_read.py b/swh/deposit/api/private/deposit_read.py --- a/swh/deposit/api/private/deposit_read.py +++ b/swh/deposit/api/private/deposit_read.py @@ -67,10 +67,6 @@ """ - ADDITIONAL_CONFIG = { - "extraction_dir": ("str", "/tmp/swh-deposit/archive/"), - } - def __init__(self): super().__init__() self.extraction_dir = self.config["extraction_dir"] @@ -107,26 +103,6 @@ """ - ADDITIONAL_CONFIG = { - "provider": ( - "dict", - { - # 'provider_name': '', # those are not set since read from the - # 'provider_url': '', # deposit's client - "provider_type": "deposit_client", - "metadata": {}, - }, - ), - "tool": ( - "dict", - { - "name": "swh-deposit", - "version": "0.0.1", - "configuration": {"sword_version": "2"}, - }, - ), - } - def __init__(self): super().__init__() self.provider = self.config["provider"] diff --git a/swh/deposit/config.py b/swh/deposit/config.py --- a/swh/deposit/config.py +++ b/swh/deposit/config.py @@ -1,15 +1,16 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os -import logging -from typing import Any, Dict, Tuple +from typing import Any, Dict -from swh.core.config import SWHConfig +from swh.core import config from swh.scheduler import get_scheduler +from swh.scheduler.interface import SchedulerInterface + # IRIs (Internationalized Resource identifier) sword 2.0 specified EDIT_SE_IRI = "edit_se_iri" @@ -49,6 +50,12 @@ } +DEFAULT_CONFIG = { + "max_upload_size": 209715200, + "checks": True, +} + + def setup_django_for(platform=None, config_file=None): """Setup function for command line tools (swh.deposit.create_user) to initialize the needed db access. @@ -81,30 +88,14 @@ django.setup() -class APIConfig(SWHConfig): - """Mixin intended to enrich views with SWH configuration. +class APIConfig: + """API Configuration centralized class. This loads explicitly the configuration file out + of the SWH_CONFIG_FILENAME environment variable. """ - CONFIG_BASE_FILENAME = "deposit/server" - - DEFAULT_CONFIG = { - "max_upload_size": ("int", 209715200), - "checks": ("bool", True), - "scheduler": ( - "dict", - {"cls": "remote", "args": {"url": "http://localhost:5008/"}}, - ), - } - - ADDITIONAL_CONFIG = {} # type: Dict[str, Tuple[str, Any]] - - def __init__(self, **config): - super().__init__() - self.config = self.parse_config_file( - additional_configs=[self.ADDITIONAL_CONFIG] - ) - self.config.update(config) - self.log = logging.getLogger("swh.deposit") - if self.config.get("scheduler"): - self.scheduler = get_scheduler(**self.config["scheduler"]) + def __init__(self): + config_file = os.environ["SWH_CONFIG_FILENAME"] + conf = config.read_raw_config(config.config_basepath(config_file)) + self.config: Dict[str, Any] = config.merge_configs(DEFAULT_CONFIG, conf) + self.scheduler: SchedulerInterface = get_scheduler(**self.config["scheduler"]) diff --git a/swh/deposit/tests/api/test_deposit_schedule.py b/swh/deposit/tests/api/test_deposit_schedule.py --- a/swh/deposit/tests/api/test_deposit_schedule.py +++ b/swh/deposit/tests/api/test_deposit_schedule.py @@ -3,9 +3,10 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import copy import datetime + from io import BytesIO -from typing import Dict from django.urls import reverse import pytest @@ -18,20 +19,13 @@ from swh.deposit.parsers import parse_xml -from ..conftest import TEST_CONFIG - - -TEST_CONFIG_WITH_CHECKS: Dict[str, object] = { - **TEST_CONFIG, - "checks": True, -} - - @pytest.fixture() -def deposit_config(): +def deposit_config(deposit_config): """Overrides the `deposit_config` fixture define in swh/deposit/tests/conftest.py to re-enable the checks.""" - return TEST_CONFIG_WITH_CHECKS + config_d = copy.deepcopy(deposit_config) + config_d["checks"] = True + return config_d def now() -> datetime.datetime: diff --git a/swh/deposit/tests/conftest.py b/swh/deposit/tests/conftest.py --- a/swh/deposit/tests/conftest.py +++ b/swh/deposit/tests/conftest.py @@ -8,6 +8,8 @@ import pytest import psycopg2 +import yaml + from django.urls import reverse from django.test.utils import setup_databases # type: ignore @@ -23,7 +25,6 @@ from swh.model.identifiers import DIRECTORY, swhid, REVISION, SNAPSHOT from swh.deposit.config import setup_django_for from swh.deposit.parsers import parse_xml -from swh.deposit.config import APIConfig from swh.deposit.config import ( COL_IRI, EDIT_SE_IRI, @@ -47,52 +48,49 @@ } -TEST_CONFIG = { - "max_upload_size": 500, - "extraction_dir": "/tmp/swh-deposit/test/extraction-dir", - "checks": False, - "provider": { - "provider_name": "", - "provider_type": "deposit_client", - "provider_url": "", - "metadata": {}, - }, - "tool": { - "name": "swh-deposit", - "version": "0.0.1", - "configuration": {"sword_version": "2"}, - }, -} - - def pytest_configure(): setup_django_for("testing") @pytest.fixture() -def deposit_config(): - return TEST_CONFIG +def deposit_config(swh_scheduler_config): + return { + "max_upload_size": 500, + "extraction_dir": "/tmp/swh-deposit/test/extraction-dir", + "checks": False, + "provider": { + "provider_name": "", + "provider_type": "deposit_client", + "provider_url": "", + "metadata": {}, + }, + "tool": { + "name": "swh-deposit", + "version": "0.0.1", + "configuration": {"sword_version": "2"}, + }, + "scheduler": {"cls": "local", "args": swh_scheduler_config,}, + } -@pytest.fixture(autouse=True) -def deposit_autoconfig(monkeypatch, deposit_config, swh_scheduler_config): - """Enforce config for deposit classes inherited from APIConfig.""" +@pytest.fixture() +def deposit_config_path(tmp_path, monkeypatch, deposit_config): + conf_path = os.path.join(tmp_path, "deposit.yml") + with open(conf_path, "w") as f: + f.write(yaml.dump(deposit_config)) + monkeypatch.setenv("SWH_CONFIG_FILENAME", conf_path) + return conf_path - def mock_parse_config(*args, **kw): - config = deposit_config.copy() - config["scheduler"] = { - "cls": "local", - "args": swh_scheduler_config, - } - return config - monkeypatch.setattr(APIConfig, "parse_config_file", mock_parse_config) +@pytest.fixture(autouse=True) +def deposit_autoconfig(deposit_config_path, swh_scheduler_config): + """Enforce config for deposit classes inherited from APIConfig.""" scheduler = get_scheduler("local", swh_scheduler_config) task_type = { "type": "load-deposit", "backend_name": "swh.loader.packages.deposit.tasks.LoadDeposit", - "description": "why does this have not-null constraint?", + "description": "Load deposit task", } scheduler.create_task_type(task_type)