diff --git a/swh/deposit/exception.py b/swh/deposit/exception.py
new file mode 100644
index 00000000..a5a32f2a
--- /dev/null
+++ b/swh/deposit/exception.py
@@ -0,0 +1,33 @@
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from typing import Dict, Optional
+
+from rest_framework.exceptions import APIException
+from rest_framework.response import Response
+from rest_framework.views import exception_handler
+
+from django.db.utils import OperationalError
+
+
+def custom_exception_handler(exc: APIException, context: Dict) -> Optional[Response]:
+ """Custom deposit exception handler to ensure consistent xml output
+
+ """
+ # drf's default exception handler first, to get the standard error response
+ response = exception_handler(exc, context)
+
+ if isinstance(exc, OperationalError):
+ status = "Database backend maintenance"
+ detail = "Service temporarily unavailable, try again later."
+ data = f"""
+
+ {status}
+ {detail}
+
+"""
+ return Response(data, status=503, content_type="application/xml")
+
+ return response
diff --git a/swh/deposit/settings/common.py b/swh/deposit/settings/common.py
index 2787315d..659651e0 100644
--- a/swh/deposit/settings/common.py
+++ b/swh/deposit/settings/common.py
@@ -1,113 +1,114 @@
-# Copyright (C) 2017 The Software Heritage developers
+# Copyright (C) 2017-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""
Django settings for swh project.
Generated by 'django-admin startproject' using Django 1.10.7.
For more information on this file, see
https://docs.djangoproject.com/en/1.10/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""
import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
ALLOWED_HOSTS = ["127.0.0.1", "localhost"]
# Application definition
INSTALLED_APPS = [
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.staticfiles",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.postgres", # for JSONField, ArrayField
"swh.deposit.apps.DepositConfig",
]
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
"swh.deposit.auth.WrapBasicAuthenticationResponseMiddleware",
]
ROOT_URLCONF = "swh.deposit.urls"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
],
},
},
]
# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", # noqa
},
{"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",},
{"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",},
{"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",},
]
# Internationalization
# https://docs.djangoproject.com/en/1.10/topics/i18n/
LANGUAGE_CODE = "en-us"
TIME_ZONE = "UTC"
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.10/howto/static-files/
STATIC_URL = "/static/"
REST_FRAMEWORK = {
"DEFAULT_AUTHENTICATION_CLASSES": (
"rest_framework.authentication.BasicAuthentication",
),
+ "EXCEPTION_HANDLER": "swh.deposit.exception.custom_exception_handler",
}
FILE_UPLOAD_HANDLERS = [
"django.core.files.uploadhandler.MemoryFileUploadHandler",
"django.core.files.uploadhandler.TemporaryFileUploadHandler",
]
diff --git a/swh/deposit/tests/api/test_exception.py b/swh/deposit/tests/api/test_exception.py
new file mode 100644
index 00000000..a3716929
--- /dev/null
+++ b/swh/deposit/tests/api/test_exception.py
@@ -0,0 +1,51 @@
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from swh.deposit.exception import custom_exception_handler
+
+from rest_framework.exceptions import APIException
+from rest_framework.response import Response
+
+from django.db.utils import OperationalError
+
+
+def test_custom_exception_handler_operational_error(mocker):
+ """Operation error are translated to service unavailable
+
+ """
+ fake_exception = OperationalError("Fake internal error", 503)
+
+ response = custom_exception_handler(fake_exception, {})
+
+ assert response is not None
+ assert response.content_type == "application/xml"
+ assert (
+ response.data
+ == """
+
+ Database backend maintenance
+ Service temporarily unavailable, try again later.
+
+"""
+ )
+
+ assert response.status_code == 503
+
+
+def test_custom_exception_handler_default_behavior_maintained(mocker):
+ """Other internal errors are transmitted as is
+
+ """
+ fake_exception = APIException("Fake internal error", 500)
+ fake_response = Response(
+ exception=fake_exception, status=fake_exception.status_code
+ )
+ mock_exception_handler = mocker.patch("swh.deposit.exception.exception_handler")
+ mock_exception_handler.return_value = fake_response
+
+ response = custom_exception_handler(fake_exception, {})
+
+ assert response is not None
+ assert response == fake_response
diff --git a/swh/deposit/tests/cli/test_client.py b/swh/deposit/tests/cli/test_client.py
index 6110bedf..c788afeb 100644
--- a/swh/deposit/tests/cli/test_client.py
+++ b/swh/deposit/tests/cli/test_client.py
@@ -1,394 +1,468 @@
-# Copyright (C) 2019 The Software Heritage developers
+# Copyright (C) 2019-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import contextlib
import logging
import os
import re
from unittest.mock import MagicMock
from click.testing import CliRunner
import pytest
from swh.deposit.client import PublicApiDepositClient
from swh.deposit.cli.client import generate_slug, _url, _client, _collection, InputError
from swh.deposit.cli import deposit as cli
from ..conftest import TEST_USER
EXAMPLE_SERVICE_DOCUMENT = {
"service": {"workspace": {"collection": {"sword:name": "softcol",}}}
}
@pytest.fixture
def slug():
return generate_slug()
@pytest.fixture
def client_mock(mocker, slug):
+ """A succesfull deposit client with hard-coded default values
+
+ """
+ mocker.patch("swh.deposit.cli.client.generate_slug", return_value=slug)
+ mock_client = MagicMock()
+ mocker.patch("swh.deposit.cli.client._client", return_value=mock_client)
+ mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT
+ mock_client.deposit_create.return_value = '{"foo": "bar"}'
+ return mock_client
+
+
+@pytest.fixture
+def client_mock_down(mocker, slug):
mocker.patch("swh.deposit.cli.client.generate_slug", return_value=slug)
mock_client = MagicMock()
mocker.patch("swh.deposit.cli.client._client", return_value=mock_client)
mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT
mock_client.deposit_create.return_value = '{"foo": "bar"}'
return mock_client
def test_url():
assert _url("http://deposit") == "http://deposit/1"
assert _url("https://other/1") == "https://other/1"
def test_client():
client = _client("http://deposit", "user", "pass")
assert isinstance(client, PublicApiDepositClient)
def test_collection_error():
mock_client = MagicMock()
mock_client.service_document.return_value = {"error": "something went wrong"}
with pytest.raises(InputError) as e:
_collection(mock_client)
assert "Service document retrieval: something went wrong" == str(e.value)
def test_collection_ok():
mock_client = MagicMock()
mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT
collection_name = _collection(mock_client)
assert collection_name == "softcol"
+def test_deposit_with_server_ok_backend_down(
+ sample_archive, mocker, caplog, client_mock, slug, tmp_path
+):
+ """ Deposit failure due to maintenance down time should be explicit in error msg
+ """
+ metadata_path = os.path.join(tmp_path, "metadata.xml")
+ mocker.patch(
+ "swh.deposit.cli.client.tempfile.TemporaryDirectory",
+ return_value=contextlib.nullcontext(str(tmp_path)),
+ )
+
+ runner = CliRunner()
+ result = runner.invoke(
+ cli,
+ [
+ "upload",
+ "--url",
+ "mock://deposit.swh/1",
+ "--username",
+ TEST_USER["username"],
+ "--password",
+ TEST_USER["password"],
+ "--name",
+ "test-project",
+ "--archive",
+ sample_archive["path"],
+ "--author",
+ "Jane Doe",
+ ],
+ )
+
+ assert result.exit_code == 0, result.output
+ assert result.output == ""
+ assert caplog.record_tuples == [
+ ("swh.deposit.cli.client", logging.INFO, '{"foo": "bar"}'),
+ ]
+
+ client_mock.deposit_create.assert_called_once_with(
+ archive=sample_archive["path"],
+ collection="softcol",
+ in_progress=False,
+ metadata=metadata_path,
+ slug=slug,
+ )
+
+ with open(metadata_path) as fd:
+ assert (
+ fd.read()
+ == f"""\
+
+
+\ttest-project
+\t{slug}
+\t
+\t\tJane Doe
+\t
+"""
+ )
+
+
def test_single_minimal_deposit(
sample_archive, mocker, caplog, client_mock, slug, tmp_path
):
""" from:
https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit
""" # noqa
metadata_path = os.path.join(tmp_path, "metadata.xml")
mocker.patch(
"swh.deposit.cli.client.tempfile.TemporaryDirectory",
return_value=contextlib.nullcontext(str(tmp_path)),
)
runner = CliRunner()
result = runner.invoke(
cli,
[
"upload",
"--url",
"mock://deposit.swh/1",
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--name",
"test-project",
"--archive",
sample_archive["path"],
"--author",
"Jane Doe",
],
)
assert result.exit_code == 0, result.output
assert result.output == ""
assert caplog.record_tuples == [
("swh.deposit.cli.client", logging.INFO, '{"foo": "bar"}'),
]
client_mock.deposit_create.assert_called_once_with(
archive=sample_archive["path"],
collection="softcol",
in_progress=False,
metadata=metadata_path,
slug=slug,
)
with open(metadata_path) as fd:
assert (
fd.read()
== f"""\
\ttest-project
\t{slug}
\t
\t\tJane Doe
\t
"""
)
def test_metadata_validation(sample_archive, mocker, caplog, tmp_path):
""" from:
https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit
""" # noqa
slug = generate_slug()
mocker.patch("swh.deposit.cli.client.generate_slug", return_value=slug)
mock_client = MagicMock()
mocker.patch("swh.deposit.cli.client._client", return_value=mock_client)
mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT
mock_client.deposit_create.return_value = '{"foo": "bar"}'
metadata_path = os.path.join(tmp_path, "metadata.xml")
mocker.patch(
"swh.deposit.cli.client.tempfile.TemporaryDirectory",
return_value=contextlib.nullcontext(str(tmp_path)),
)
with open(metadata_path, "a"):
pass # creates the file
runner = CliRunner()
# Test missing author
result = runner.invoke(
cli,
[
"upload",
"--url",
"mock://deposit.swh/1",
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--name",
"test-project",
"--archive",
sample_archive["path"],
],
)
assert result.exit_code == 1, result.output
assert result.output == ""
assert len(caplog.record_tuples) == 1
(_logger, level, message) = caplog.record_tuples[0]
assert level == logging.ERROR
assert " --author " in message
# Clear mocking state
caplog.clear()
mock_client.reset_mock()
# Test missing name
result = runner.invoke(
cli,
[
"upload",
"--url",
"mock://deposit.swh/1",
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--archive",
sample_archive["path"],
"--author",
"Jane Doe",
],
)
assert result.exit_code == 1, result.output
assert result.output == ""
assert len(caplog.record_tuples) == 1
(_logger, level, message) = caplog.record_tuples[0]
assert level == logging.ERROR
assert " --name " in message
# Clear mocking state
caplog.clear()
mock_client.reset_mock()
# Test both --metadata and --author
result = runner.invoke(
cli,
[
"upload",
"--url",
"mock://deposit.swh/1",
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--archive",
sample_archive["path"],
"--metadata",
metadata_path,
"--author",
"Jane Doe",
],
)
assert result.exit_code == 1, result.output
assert result.output == ""
assert len(caplog.record_tuples) == 1
(_logger, level, message) = caplog.record_tuples[0]
assert level == logging.ERROR
assert re.search("--metadata.*is incompatible with", message)
# Clear mocking state
caplog.clear()
mock_client.reset_mock()
def test_single_deposit_slug_generation(
sample_archive, mocker, caplog, tmp_path, client_mock
):
""" from:
https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit
""" # noqa
slug = "my-slug"
collection = "my-collection"
metadata_path = os.path.join(tmp_path, "metadata.xml")
mocker.patch(
"swh.deposit.cli.client.tempfile.TemporaryDirectory",
return_value=contextlib.nullcontext(str(tmp_path)),
)
runner = CliRunner()
result = runner.invoke(
cli,
[
"upload",
"--url",
"mock://deposit.swh/1",
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--name",
"test-project",
"--archive",
sample_archive["path"],
"--slug",
slug,
"--collection",
collection,
"--author",
"Jane Doe",
],
)
assert result.exit_code == 0, result.output
assert result.output == ""
assert caplog.record_tuples == [
("swh.deposit.cli.client", logging.INFO, '{"foo": "bar"}'),
]
client_mock.deposit_create.assert_called_once_with(
archive=sample_archive["path"],
collection=collection,
in_progress=False,
metadata=metadata_path,
slug=slug,
)
with open(metadata_path) as fd:
assert (
fd.read()
== """\
\ttest-project
\tmy-slug
\t
\t\tJane Doe
\t
"""
)
def test_multisteps_deposit(
sample_archive, atom_dataset, mocker, caplog, datadir, client_mock, slug
):
""" from:
https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#multisteps-deposit
""" # noqa
slug = generate_slug()
mocker.patch("swh.deposit.cli.client.generate_slug", return_value=slug)
# https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#create-an-incomplete-deposit
client_mock.deposit_create.return_value = '{"deposit_id": "42"}'
runner = CliRunner()
result = runner.invoke(
cli,
[
"upload",
"--url",
"mock://deposit.swh/1",
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--archive",
sample_archive["path"],
"--partial",
],
)
assert result.exit_code == 0, result.output
assert result.output == ""
assert caplog.record_tuples == [
("swh.deposit.cli.client", logging.INFO, '{"deposit_id": "42"}'),
]
client_mock.deposit_create.assert_called_once_with(
archive=sample_archive["path"],
collection="softcol",
in_progress=True,
metadata=None,
slug=slug,
)
# Clear mocking state
caplog.clear()
client_mock.reset_mock()
# https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#add-content-or-metadata-to-the-deposit
metadata_path = os.path.join(datadir, "atom", "entry-data-deposit-binary.xml")
result = runner.invoke(
cli,
[
"upload",
"--url",
"mock://deposit.swh/1",
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--metadata",
metadata_path,
],
)
assert result.exit_code == 0, result.output
assert result.output == ""
assert caplog.record_tuples == [
("swh.deposit.cli.client", logging.INFO, '{"deposit_id": "42"}'),
]
client_mock.deposit_create.assert_called_once_with(
archive=None,
collection="softcol",
in_progress=False,
metadata=metadata_path,
slug=slug,
)
# Clear mocking state
caplog.clear()
client_mock.reset_mock()