diff --git a/swh/deposit/tests/cli/test_client.py b/swh/deposit/tests/cli/test_client.py
index ca92907c..2d3fb2ae 100644
--- a/swh/deposit/tests/cli/test_client.py
+++ b/swh/deposit/tests/cli/test_client.py
@@ -1,468 +1,442 @@
# Copyright (C) 2019-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import contextlib
+import json
import logging
import os
from unittest.mock import MagicMock
from click.testing import CliRunner
import pytest
from swh.deposit.cli import deposit as cli
from swh.deposit.cli.client import InputError, _client, _collection, _url, generate_slug
from swh.deposit.client import MaintenanceError, PublicApiDepositClient
from swh.deposit.parsers import parse_xml
from ..conftest import TEST_USER
EXAMPLE_SERVICE_DOCUMENT = {
"service": {"workspace": {"collection": {"sword:name": "softcol",}}}
}
@pytest.fixture
def deposit_config():
return {
"url": "https://deposit.swh.test/1",
"auth": {"username": "test", "password": "test",},
}
@pytest.fixture
def datadir(request):
"""Override default datadir to target main test datadir"""
return os.path.join(os.path.dirname(str(request.fspath)), "../data")
@pytest.fixture
def slug():
return generate_slug()
-@pytest.fixture
-def client_mock(mocker, slug):
- """A successful deposit client with hard-coded default values
-
- """
- mocker.patch("swh.deposit.cli.client.generate_slug", return_value=slug)
- mock_client = MagicMock()
- mocker.patch("swh.deposit.cli.client._client", return_value=mock_client)
- mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT
- mock_client.deposit_create.return_value = '{"foo": "bar"}'
- return mock_client
-
-
@pytest.fixture
def client_mock_api_down(mocker, slug):
"""A mock client whose connection with api fails due to maintenance issue
"""
mocker.patch("swh.deposit.cli.client.generate_slug", return_value=slug)
mock_client = MagicMock()
mocker.patch("swh.deposit.cli.client._client", return_value=mock_client)
mock_client.service_document.side_effect = MaintenanceError(
"Database backend maintenance: Temporarily unavailable, try again later."
)
return mock_client
def test_url():
assert _url("http://deposit") == "http://deposit/1"
assert _url("https://other/1") == "https://other/1"
def test_client():
client = _client("http://deposit", "user", "pass")
assert isinstance(client, PublicApiDepositClient)
def test_collection_error():
mock_client = MagicMock()
mock_client.service_document.return_value = {"error": "something went wrong"}
with pytest.raises(InputError) as e:
_collection(mock_client)
assert "Service document retrieval: something went wrong" == str(e.value)
def test_collection_ok(deposit_config, requests_mock_datadir):
client = PublicApiDepositClient(deposit_config)
collection_name = _collection(client)
assert collection_name == "test"
def test_collection_ko_because_downtime():
mock_client = MagicMock()
mock_client.service_document.side_effect = MaintenanceError("downtime")
with pytest.raises(MaintenanceError, match="downtime"):
_collection(mock_client)
def test_deposit_with_server_down_for_maintenance(
sample_archive, mocker, caplog, client_mock_api_down, slug, tmp_path
):
""" Deposit failure due to maintenance down time should be explicit
"""
runner = CliRunner()
result = runner.invoke(
cli,
[
"upload",
"--url",
"https://deposit.swh.test/1",
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--name",
"test-project",
"--archive",
sample_archive["path"],
"--author",
"Jane Doe",
],
)
assert result.exit_code == 1, result.output
assert result.output == ""
assert caplog.record_tuples == [
(
"swh.deposit.cli.client",
logging.ERROR,
"Database backend maintenance: Temporarily unavailable, try again later.",
)
]
client_mock_api_down.service_document.assert_called_once_with()
def test_single_minimal_deposit(
sample_archive, mocker, caplog, slug, tmp_path, requests_mock_datadir
):
""" This ensure a single deposit upload through the cli is fine, cf.
https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit
""" # noqa
metadata_path = os.path.join(tmp_path, "metadata.xml")
mocker.patch(
"tempfile.TemporaryDirectory",
return_value=contextlib.nullcontext(str(tmp_path)),
)
runner = CliRunner()
result = runner.invoke(
cli,
[
"upload",
"--url",
"https://deposit.swh.test/1",
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--name",
"test-project",
"--archive",
sample_archive["path"],
"--author",
"Jane Doe",
"--slug",
slug,
],
)
assert result.exit_code == 0, result.output
assert result.output == ""
interesting_records = []
for record in caplog.record_tuples:
if record[0] == "swh.deposit.cli.client":
interesting_records.append(record)
assert len(interesting_records) == 1
assert interesting_records == [
(
"swh.deposit.cli.client",
logging.INFO,
"{'deposit_id': '615', 'deposit_status': 'partial', 'deposit_status_detail': None, 'deposit_date': 'Oct. 8, 2020, 4:57 p.m.'}", # noqa
),
]
with open(metadata_path) as fd:
assert (
fd.read()
== f"""\
\ttest-project
\t{slug}
\t
\t\tJane Doe
\t
"""
)
def test_metadata_validation(
sample_archive, mocker, caplog, tmp_path, requests_mock_datadir
):
"""Multiple metadata flags scenario (missing, conflicts) properly fails the calls
"""
slug = generate_slug()
metadata_path = os.path.join(tmp_path, "metadata.xml")
mocker.patch(
"tempfile.TemporaryDirectory",
return_value=contextlib.nullcontext(str(tmp_path)),
)
with open(metadata_path, "a"):
pass # creates the file
runner = CliRunner()
# Test missing author
result = runner.invoke(
cli,
[
"upload",
"--url",
"https://deposit.swh.test/1",
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--name",
"test-project",
"--archive",
sample_archive["path"],
"--slug",
slug,
],
)
assert result.exit_code == 1, result.output
assert result.output == ""
assert len(caplog.record_tuples) == 1
(_logger, level, message) = caplog.record_tuples[0]
assert level == logging.ERROR
assert message == (
"Problem during parsing options: Either a metadata file"
" (--metadata) or both --author and --name must be provided, "
"unless this is an archive-only deposit."
)
# Clear mocking state
caplog.clear()
# Test missing name
result = runner.invoke(
cli,
[
"upload",
"--url",
"https://deposit.swh.test/1",
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--archive",
sample_archive["path"],
"--author",
"Jane Doe",
"--slug",
slug,
],
)
assert result.exit_code == 1, result.output
assert result.output == ""
assert len(caplog.record_tuples) == 1
(_logger, level, message) = caplog.record_tuples[0]
assert level == logging.ERROR
assert message == (
"Problem during parsing options: Either a metadata file"
" (--metadata) or both --author and --name must be provided, "
"unless this is an archive-only deposit."
)
# Clear mocking state
caplog.clear()
# Test both --metadata and --author
result = runner.invoke(
cli,
[
"upload",
"--url",
"https://deposit.swh.test/1",
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--archive",
sample_archive["path"],
"--metadata",
metadata_path,
"--author",
"Jane Doe",
"--slug",
slug,
],
)
assert result.exit_code == 1, result.output
assert result.output == ""
assert len(caplog.record_tuples) == 1
(_logger, level, message) = caplog.record_tuples[0]
assert level == logging.ERROR
assert message == (
"Problem during parsing options: Using a metadata file "
"(--metadata) is incompatible with --author and --name, "
"which are used to generate one."
)
def test_single_deposit_slug_generation(
sample_archive, mocker, caplog, tmp_path, requests_mock_datadir
):
"""Single deposit scenario without providing the slug, the slug is generated nonetheless
https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit
""" # noqa
metadata_path = os.path.join(tmp_path, "metadata.xml")
mocker.patch(
"tempfile.TemporaryDirectory",
return_value=contextlib.nullcontext(str(tmp_path)),
)
runner = CliRunner()
result = runner.invoke(
cli,
[
"upload",
"--url",
"https://deposit.swh.test/1",
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--name",
"test-project",
"--archive",
sample_archive["path"],
"--author",
"Jane Doe",
],
)
assert result.exit_code == 0, result.output
assert result.output == ""
interesting_records = []
for record in caplog.record_tuples:
if record[0] == "swh.deposit.cli.client":
interesting_records.append(record)
assert len(interesting_records) == 1
assert interesting_records == [
(
"swh.deposit.cli.client",
logging.INFO,
"{'deposit_id': '615', 'deposit_status': 'partial', 'deposit_status_detail': None, 'deposit_date': 'Oct. 8, 2020, 4:57 p.m.'}", # noqa
),
]
with open(metadata_path) as fd:
metadata_xml = fd.read()
actual_metadata = parse_xml(metadata_xml)
assert actual_metadata["codemeta:identifier"] is not None
-def test_multisteps_deposit(
- sample_archive, atom_dataset, mocker, caplog, datadir, client_mock, slug
-):
- """ from:
+def test_multisteps_deposit(sample_archive, datadir, slug, requests_mock_datadir):
+ """ First deposit a partial deposit (no metadata, only archive), then update the metadata part.
https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#multisteps-deposit
""" # noqa
- slug = generate_slug()
- mocker.patch("swh.deposit.cli.client.generate_slug", return_value=slug)
-
- # https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#create-an-incomplete-deposit
- client_mock.deposit_create.return_value = '{"deposit_id": "42"}'
+ api_url = "https://deposit.test.metadata/1"
runner = CliRunner()
result = runner.invoke(
cli,
[
"upload",
"--url",
- "mock://deposit.swh/1",
+ api_url,
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--archive",
sample_archive["path"],
"--partial",
+ "--slug",
+ slug,
+ "--format",
+ "json",
],
)
- assert result.exit_code == 0, result.output
- assert result.output == ""
- assert caplog.record_tuples == [
- ("swh.deposit.cli.client", logging.INFO, '{"deposit_id": "42"}'),
- ]
-
- client_mock.deposit_create.assert_called_once_with(
- archive=sample_archive["path"],
- collection="softcol",
- in_progress=True,
- metadata=None,
- slug=slug,
- )
-
- # Clear mocking state
- caplog.clear()
- client_mock.reset_mock()
+ assert result.exit_code == 0, f"unexpected output: {result.output}"
+ actual_deposit = json.loads(result.output)
+ assert actual_deposit == {
+ "deposit_id": "666",
+ "deposit_status": "partial",
+ "deposit_status_detail": None,
+ "deposit_date": "Oct. 8, 2020, 4:57 p.m.",
+ }
# https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#add-content-or-metadata-to-the-deposit
-
metadata_path = os.path.join(datadir, "atom", "entry-data-deposit-binary.xml")
+ # Update deposit with metadata
result = runner.invoke(
cli,
[
"upload",
"--url",
- "mock://deposit.swh/1",
+ api_url,
"--username",
TEST_USER["username"],
"--password",
TEST_USER["password"],
"--metadata",
metadata_path,
+ "--deposit-id",
+ 666,
+ "--slug",
+ slug,
+ "--format",
+ "json",
],
)
- assert result.exit_code == 0, result.output
- assert result.output == ""
- assert caplog.record_tuples == [
- ("swh.deposit.cli.client", logging.INFO, '{"deposit_id": "42"}'),
- ]
-
- client_mock.deposit_create.assert_called_once_with(
- archive=None,
- collection="softcol",
- in_progress=False,
- metadata=metadata_path,
- slug=slug,
- )
-
- # Clear mocking state
- caplog.clear()
- client_mock.reset_mock()
+ assert result.exit_code == 0, f"unexpected output: {result.output}"
+ assert result.output is not None
+ actual_deposit = json.loads(result.output)
+ # deposit update scenario actually returns a deposit status dict
+ assert actual_deposit["deposit_id"] == "666"
+ # FIXME: should be "deposited" but current limitation in the
+ # requests_mock_datadir_visits use, cannot find a way to make it work right now
+ assert actual_deposit["deposit_status"] == "partial"
diff --git a/swh/deposit/tests/data/https_deposit.test.metadata/1_servicedocument b/swh/deposit/tests/data/https_deposit.test.metadata/1_servicedocument
new file mode 100644
index 00000000..033ca6ea
--- /dev/null
+++ b/swh/deposit/tests/data/https_deposit.test.metadata/1_servicedocument
@@ -0,0 +1,26 @@
+
+
+
+ 2.0
+ 209715200
+
+
+ The Software Heritage (SWH) Archive
+
+ test Software Collection
+ application/zip
+ application/x-tar
+ Collection Policy
+ Software Heritage Archive
+ Collect, Preserve, Share
+ false
+ false
+ http://purl.org/net/sword/package/SimpleZip
+ https://deposit.test.metadata/1/test/
+ test
+
+
+
diff --git a/swh/deposit/tests/data/https_deposit.test.metadata/1_test b/swh/deposit/tests/data/https_deposit.test.metadata/1_test
new file mode 100644
index 00000000..9622a83f
--- /dev/null
+++ b/swh/deposit/tests/data/https_deposit.test.metadata/1_test
@@ -0,0 +1,19 @@
+
+ 666
+ Oct. 8, 2020, 4:57 p.m.
+ hardcoded_sample_archive_path
+ partial
+
+
+
+
+
+
+
+
+
+
+ http://purl.org/net/sword/package/SimpleZip
+
diff --git a/swh/deposit/tests/data/https_deposit.test.metadata/1_test_666_metadata b/swh/deposit/tests/data/https_deposit.test.metadata/1_test_666_metadata
new file mode 100644
index 00000000..6a2c89e3
--- /dev/null
+++ b/swh/deposit/tests/data/https_deposit.test.metadata/1_test_666_metadata
@@ -0,0 +1,19 @@
+
+ 666
+ Oct. 9, 2020, 8:44 p.m.
+ something
+ deposited
+
+
+
+
+
+
+
+
+
+
+ http://purl.org/net/sword/package/SimpleZip
+
diff --git a/swh/deposit/tests/data/https_deposit.test.metadata/1_test_666_status b/swh/deposit/tests/data/https_deposit.test.metadata/1_test_666_status
new file mode 100644
index 00000000..0af5ba9e
--- /dev/null
+++ b/swh/deposit/tests/data/https_deposit.test.metadata/1_test_666_status
@@ -0,0 +1,8 @@
+
+ 666
+ partial
+ Deposit is partially received. To finalize it, In-Progress header should be false
+ external-id
+