diff --git a/requirements-test.txt b/requirements-test.txt --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,6 @@ pytest pytest-django +pytest-mock swh.scheduler[testing] pytest-postgresql >= 2.1.0 requests_mock diff --git a/swh/deposit/cli/client.py b/swh/deposit/cli/client.py --- a/swh/deposit/cli/client.py +++ b/swh/deposit/cli/client.py @@ -5,6 +5,7 @@ import os import logging +import sys import tempfile import uuid @@ -335,12 +336,8 @@ metadata_deposit, collection, slug, partial, deposit_id, replace, url, name, author) except InputError as e: - msg = 'Problem during parsing options: %s' % e - r = { - 'error': msg, - } - logger.info(r) - return 1 + logger.error('Problem during parsing options: %s', e) + sys.exit(1) try: if verbose: @@ -382,12 +379,8 @@ client = _client(url, username, password) collection = _collection(client) except InputError as e: - msg = 'Problem during parsing options: %s' % e - r = { - 'error': msg, - } - logger.info(r) - return 1 + logger.error('Problem during parsing options: %s', e) + sys.exit(1) r = client.deposit_status( collection=collection, deposit_id=deposit_id) diff --git a/swh/deposit/tests/api/conftest.py b/swh/deposit/tests/api/conftest.py --- a/swh/deposit/tests/api/conftest.py +++ b/swh/deposit/tests/api/conftest.py @@ -7,8 +7,6 @@ import pytest from django.urls import reverse -from os import path, listdir -from typing import Mapping from swh.deposit.config import ( DEPOSIT_STATUS_DEPOSITED, COL_IRI, DEPOSIT_STATUS_VERIFIED @@ -19,28 +17,6 @@ from swh.deposit.api.private.deposit_check import SWHChecksDeposit -@pytest.fixture -def atom_dataset(datadir) -> Mapping[str, bytes]: - """Compute the paths to atom files. - - Returns: - Dict of atom name per content (bytes) - - """ - atom_path = path.join(datadir, 'atom') - data = {} - for filename in listdir(atom_path): - filepath = path.join(atom_path, filename) - with open(filepath, 'rb') as f: - raw_content = f.read() - - # Keep the filename without extension - atom_name = filename.split('.')[0] - data[atom_name] = raw_content - - return data - - @pytest.fixture def ready_deposit_ok(partial_deposit_with_metadata): """Returns a deposit ready for checks (it will pass the checks). diff --git a/swh/deposit/tests/api/data/atom b/swh/deposit/tests/api/data/atom new file mode 120000 --- /dev/null +++ b/swh/deposit/tests/api/data/atom @@ -0,0 +1 @@ +../../data/atom \ No newline at end of file diff --git a/swh/deposit/tests/api/test_service_document.py b/swh/deposit/tests/api/test_service_document.py --- a/swh/deposit/tests/api/test_service_document.py +++ b/swh/deposit/tests/api/test_service_document.py @@ -54,19 +54,23 @@ def check_response(response, username): assert response.status_code == status.HTTP_200_OK assert response.content.decode('utf-8') == \ - ''' + get_expected_service_response('http://testserver/1', username) + + +def get_expected_service_response(base_url, username): + return f''' 2.0 - %s + {TEST_CONFIG['max_upload_size']} The Software Heritage (SWH) Archive - - %s Software Collection + + {username} Software Collection application/zip application/x-tar Collection Policy @@ -75,13 +79,9 @@ false false http://purl.org/net/sword/package/SimpleZip - http://testserver/1/%s/ - %s + {base_url}/{username}/ + {username} -''' % (TEST_CONFIG['max_upload_size'], - username, - username, - username, - username) # noqa +''' # noqa diff --git a/swh/deposit/tests/cli/data/atom b/swh/deposit/tests/cli/data/atom new file mode 120000 --- /dev/null +++ b/swh/deposit/tests/cli/data/atom @@ -0,0 +1 @@ +../../data/atom \ No newline at end of file diff --git a/swh/deposit/tests/cli/test_client.py b/swh/deposit/tests/cli/test_client.py --- a/swh/deposit/tests/cli/test_client.py +++ b/swh/deposit/tests/cli/test_client.py @@ -3,12 +3,29 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import pytest - +import logging +import os from unittest.mock import MagicMock +from click.testing import CliRunner +import pytest + from swh.deposit.client import PublicApiDepositClient -from swh.deposit.cli.client import _url, _client, _collection, InputError +from swh.deposit.cli.client import ( + generate_slug, _url, _client, _collection, InputError) +from swh.deposit.cli import deposit as cli +from ..conftest import TEST_USER + + +EXAMPLE_SERVICE_DOCUMENT = { + 'service': { + 'workspace': { + 'collection': { + 'sword:name': 'softcol', + } + } + } +} def test_url(): @@ -35,15 +52,149 @@ def test_collection_ok(): mock_client = MagicMock() - mock_client.service_document.return_value = { - 'service': { - 'workspace': { - 'collection': { - 'sword:name': 'softcol', - } - } - } - } + mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT collection_name = _collection(mock_client) assert collection_name == 'softcol' + + +def test_single_minimal_deposit(sample_archive, mocker, caplog): + """ from: + https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit + """ # noqa + slug = generate_slug() + mocker.patch('swh.deposit.cli.client.generate_slug', return_value=slug) + mock_client = MagicMock() + mocker.patch( + 'swh.deposit.cli.client._client', + return_value=mock_client) + mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT + mock_client.deposit_create.return_value = '{"foo": "bar"}' + + runner = CliRunner() + result = runner.invoke(cli, [ + 'upload', + '--url', 'mock://deposit.swh/1', + '--username', TEST_USER['username'], + '--password', TEST_USER['password'], + '--name', 'test-project', + '--archive', sample_archive['path'], + ]) + + assert result.exit_code == 0, result.output + assert result.output == '' + assert caplog.record_tuples == [ + ('swh.deposit.cli.client', logging.INFO, '{"foo": "bar"}'), + ] + + mock_client.deposit_create.assert_called_once_with( + archive=sample_archive['path'], + collection='softcol', in_progress=False, metadata=None, + slug=slug) + + +def test_single_deposit_slug_collection(sample_archive, mocker, caplog): + """ from: + https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit + """ # noqa + slug = 'my-slug' + collection = 'my-collection' + mock_client = MagicMock() + mocker.patch( + 'swh.deposit.cli.client._client', + return_value=mock_client) + mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT + mock_client.deposit_create.return_value = '{"foo": "bar"}' + + runner = CliRunner() + result = runner.invoke(cli, [ + 'upload', + '--url', 'mock://deposit.swh/1', + '--username', TEST_USER['username'], + '--password', TEST_USER['password'], + '--name', 'test-project', + '--archive', sample_archive['path'], + '--slug', slug, + '--collection', collection, + ]) + + assert result.exit_code == 0, result.output + assert result.output == '' + assert caplog.record_tuples == [ + ('swh.deposit.cli.client', logging.INFO, '{"foo": "bar"}'), + ] + + mock_client.deposit_create.assert_called_once_with( + archive=sample_archive['path'], + collection=collection, in_progress=False, metadata=None, + slug=slug) + + +def test_multisteps_deposit( + sample_archive, atom_dataset, mocker, caplog, datadir): + """ from: + https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#multisteps-deposit + """ # noqa + slug = generate_slug() + mocker.patch('swh.deposit.cli.client.generate_slug', return_value=slug) + mock_client = MagicMock() + mocker.patch( + 'swh.deposit.cli.client._client', + return_value=mock_client) + + # https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#create-an-incomplete-deposit + mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT + mock_client.deposit_create.return_value = '{"deposit_id": "42"}' + + runner = CliRunner() + result = runner.invoke(cli, [ + 'upload', + '--url', 'mock://deposit.swh/1', + '--username', TEST_USER['username'], + '--password', TEST_USER['password'], + '--archive', sample_archive['path'], + '--partial', + ]) + + assert result.exit_code == 0, result.output + assert result.output == '' + assert caplog.record_tuples == [ + ('swh.deposit.cli.client', logging.INFO, '{"deposit_id": "42"}'), + ] + + mock_client.deposit_create.assert_called_once_with( + archive=sample_archive['path'], + collection='softcol', in_progress=True, metadata=None, + slug=slug) + + # Clear mocking state + caplog.clear() + mock_client.reset_mock() + + # https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#add-content-or-metadata-to-the-deposit + + metadata_path = os.path.join( + datadir, 'atom', 'entry-data-deposit-binary.xml') + + result = runner.invoke(cli, [ + 'upload', + '--url', 'mock://deposit.swh/1', + '--username', TEST_USER['username'], + '--password', TEST_USER['password'], + '--metadata', metadata_path, + ]) + + assert result.exit_code == 0, result.output + assert result.output == '' + assert caplog.record_tuples == [ + ('swh.deposit.cli.client', logging.INFO, '{"deposit_id": "42"}'), + ] + + mock_client.deposit_create.assert_called_once_with( + archive=None, + collection='softcol', in_progress=False, metadata=metadata_path, + slug=slug) + + # Clear mocking state + caplog.clear() + mock_client.reset_mock() diff --git a/swh/deposit/tests/conftest.py b/swh/deposit/tests/conftest.py --- a/swh/deposit/tests/conftest.py +++ b/swh/deposit/tests/conftest.py @@ -163,6 +163,28 @@ return archive +@pytest.fixture +def atom_dataset(datadir) -> Mapping[str, bytes]: + """Compute the paths to atom files. + + Returns: + Dict of atom name per content (bytes) + + """ + atom_path = os.path.join(datadir, 'atom') + data = {} + for filename in os.listdir(atom_path): + filepath = os.path.join(atom_path, filename) + with open(filepath, 'rb') as f: + raw_content = f.read() + + # Keep the filename without extension + atom_name = filename.split('.')[0] + data[atom_name] = raw_content + + return data + + def create_deposit( authenticated_client, collection_name: str, sample_archive, external_id: str, deposit_status=DEPOSIT_STATUS_DEPOSITED): diff --git a/swh/deposit/tests/api/data/atom/codemeta-sample.xml b/swh/deposit/tests/data/atom/codemeta-sample.xml rename from swh/deposit/tests/api/data/atom/codemeta-sample.xml rename to swh/deposit/tests/data/atom/codemeta-sample.xml diff --git a/swh/deposit/tests/api/data/atom/entry-data-badly-formatted.xml b/swh/deposit/tests/data/atom/entry-data-badly-formatted.xml rename from swh/deposit/tests/api/data/atom/entry-data-badly-formatted.xml rename to swh/deposit/tests/data/atom/entry-data-badly-formatted.xml diff --git a/swh/deposit/tests/api/data/atom/entry-data-deposit-binary.xml b/swh/deposit/tests/data/atom/entry-data-deposit-binary.xml rename from swh/deposit/tests/api/data/atom/entry-data-deposit-binary.xml rename to swh/deposit/tests/data/atom/entry-data-deposit-binary.xml diff --git a/swh/deposit/tests/api/data/atom/entry-data-empty-body.xml b/swh/deposit/tests/data/atom/entry-data-empty-body.xml rename from swh/deposit/tests/api/data/atom/entry-data-empty-body.xml rename to swh/deposit/tests/data/atom/entry-data-empty-body.xml diff --git a/swh/deposit/tests/api/data/atom/entry-data-ko.xml b/swh/deposit/tests/data/atom/entry-data-ko.xml rename from swh/deposit/tests/api/data/atom/entry-data-ko.xml rename to swh/deposit/tests/data/atom/entry-data-ko.xml diff --git a/swh/deposit/tests/api/data/atom/entry-data-minimal.xml b/swh/deposit/tests/data/atom/entry-data-minimal.xml rename from swh/deposit/tests/api/data/atom/entry-data-minimal.xml rename to swh/deposit/tests/data/atom/entry-data-minimal.xml diff --git a/swh/deposit/tests/api/data/atom/entry-data-parsing-error-prone.xml b/swh/deposit/tests/data/atom/entry-data-parsing-error-prone.xml rename from swh/deposit/tests/api/data/atom/entry-data-parsing-error-prone.xml rename to swh/deposit/tests/data/atom/entry-data-parsing-error-prone.xml diff --git a/swh/deposit/tests/api/data/atom/entry-data0.xml b/swh/deposit/tests/data/atom/entry-data0.xml rename from swh/deposit/tests/api/data/atom/entry-data0.xml rename to swh/deposit/tests/data/atom/entry-data0.xml diff --git a/swh/deposit/tests/api/data/atom/entry-data1.xml b/swh/deposit/tests/data/atom/entry-data1.xml rename from swh/deposit/tests/api/data/atom/entry-data1.xml rename to swh/deposit/tests/data/atom/entry-data1.xml diff --git a/swh/deposit/tests/api/data/atom/entry-data2.xml b/swh/deposit/tests/data/atom/entry-data2.xml rename from swh/deposit/tests/api/data/atom/entry-data2.xml rename to swh/deposit/tests/data/atom/entry-data2.xml diff --git a/swh/deposit/tests/api/data/atom/entry-data3.xml b/swh/deposit/tests/data/atom/entry-data3.xml rename from swh/deposit/tests/api/data/atom/entry-data3.xml rename to swh/deposit/tests/data/atom/entry-data3.xml diff --git a/swh/deposit/tests/api/data/atom/entry-update-in-place.xml b/swh/deposit/tests/data/atom/entry-update-in-place.xml rename from swh/deposit/tests/api/data/atom/entry-update-in-place.xml rename to swh/deposit/tests/data/atom/entry-update-in-place.xml diff --git a/swh/deposit/tests/api/data/atom/error-with-decimal.xml b/swh/deposit/tests/data/atom/error-with-decimal.xml rename from swh/deposit/tests/api/data/atom/error-with-decimal.xml rename to swh/deposit/tests/data/atom/error-with-decimal.xml diff --git a/swh/deposit/tests/api/data/atom/metadata.xml b/swh/deposit/tests/data/atom/metadata.xml rename from swh/deposit/tests/api/data/atom/metadata.xml rename to swh/deposit/tests/data/atom/metadata.xml diff --git a/swh/deposit/tests/api/data/atom/tei-sample.xml b/swh/deposit/tests/data/atom/tei-sample.xml rename from swh/deposit/tests/api/data/atom/tei-sample.xml rename to swh/deposit/tests/data/atom/tei-sample.xml