diff --git a/PKG-INFO b/PKG-INFO index 78f60ba3..4dd32121 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,37 +1,37 @@ Metadata-Version: 2.1 Name: swh.deposit -Version: 0.0.86 +Version: 0.0.87 Summary: Software Heritage Deposit Server Home-page: https://forge.softwareheritage.org/source/swh-deposit/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-deposit Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-deposit/ Description: # swh-deposit This is [Software Heritage](https://www.softwareheritage.org)'s [SWORD 2.0](http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html) Server implementation, as well as a simple client to upload deposits on the server. **S.W.O.R.D** (**S**imple **W**eb-Service **O**ffering **R**epository **D**eposit) is an interoperability standard for digital file deposit. This implementation will permit interaction between a client (a repository) and a server (SWH repository) to permit deposits of software source code archives and associated metadata. The documentation is at ./docs/README-specification.md Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing Provides-Extra: server diff --git a/requirements-swh-server.txt b/requirements-swh-server.txt index b12106e5..86a85993 100644 --- a/requirements-swh-server.txt +++ b/requirements-swh-server.txt @@ -1,4 +1,4 @@ swh.core[http] swh.loader.core >= 0.0.71 swh.scheduler >= 0.0.39 -swh.model >= 0.0.26 +swh.model >= 0.1.0 diff --git a/swh.deposit.egg-info/PKG-INFO b/swh.deposit.egg-info/PKG-INFO index 78f60ba3..4dd32121 100644 --- a/swh.deposit.egg-info/PKG-INFO +++ b/swh.deposit.egg-info/PKG-INFO @@ -1,37 +1,37 @@ Metadata-Version: 2.1 Name: swh.deposit -Version: 0.0.86 +Version: 0.0.87 Summary: Software Heritage Deposit Server Home-page: https://forge.softwareheritage.org/source/swh-deposit/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-deposit Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-deposit/ Description: # swh-deposit This is [Software Heritage](https://www.softwareheritage.org)'s [SWORD 2.0](http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html) Server implementation, as well as a simple client to upload deposits on the server. **S.W.O.R.D** (**S**imple **W**eb-Service **O**ffering **R**epository **D**eposit) is an interoperability standard for digital file deposit. This implementation will permit interaction between a client (a repository) and a server (SWH repository) to permit deposits of software source code archives and associated metadata. The documentation is at ./docs/README-specification.md Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing Provides-Extra: server diff --git a/swh.deposit.egg-info/SOURCES.txt b/swh.deposit.egg-info/SOURCES.txt index c0c4ccdf..6538738d 100644 --- a/swh.deposit.egg-info/SOURCES.txt +++ b/swh.deposit.egg-info/SOURCES.txt @@ -1,175 +1,176 @@ MANIFEST.in Makefile README.md pyproject.toml pytest.ini requirements-server.txt requirements-swh-server.txt requirements-swh.txt requirements-test.txt requirements.txt setup.cfg setup.py tox.ini version.txt swh/__init__.py swh.deposit.egg-info/PKG-INFO swh.deposit.egg-info/SOURCES.txt swh.deposit.egg-info/dependency_links.txt swh.deposit.egg-info/entry_points.txt swh.deposit.egg-info/requires.txt swh.deposit.egg-info/top_level.txt swh/deposit/__init__.py swh/deposit/apps.py swh/deposit/auth.py swh/deposit/client.py swh/deposit/config.py swh/deposit/errors.py swh/deposit/exception.py swh/deposit/gunicorn_config.py swh/deposit/manage.py swh/deposit/models.py swh/deposit/parsers.py swh/deposit/py.typed swh/deposit/urls.py swh/deposit/utils.py swh/deposit/api/__init__.py swh/deposit/api/common.py swh/deposit/api/converters.py swh/deposit/api/deposit.py swh/deposit/api/deposit_content.py swh/deposit/api/deposit_status.py swh/deposit/api/deposit_update.py swh/deposit/api/service_document.py swh/deposit/api/urls.py swh/deposit/api/private/__init__.py swh/deposit/api/private/deposit_check.py swh/deposit/api/private/deposit_list.py swh/deposit/api/private/deposit_read.py swh/deposit/api/private/deposit_update_status.py swh/deposit/api/private/urls.py swh/deposit/cli/__init__.py swh/deposit/cli/admin.py swh/deposit/cli/client.py swh/deposit/fixtures/__init__.py swh/deposit/fixtures/deposit_data.yaml swh/deposit/loader/__init__.py swh/deposit/loader/checker.py swh/deposit/loader/tasks.py swh/deposit/migrations/0001_initial.py swh/deposit/migrations/0002_depositrequest_archive.py swh/deposit/migrations/0003_temporaryarchive.py swh/deposit/migrations/0004_delete_temporaryarchive.py swh/deposit/migrations/0005_auto_20171019_1436.py swh/deposit/migrations/0006_depositclient_url.py swh/deposit/migrations/0007_auto_20171129_1609.py swh/deposit/migrations/0008_auto_20171130_1513.py swh/deposit/migrations/0009_deposit_parent.py swh/deposit/migrations/0010_auto_20180110_0953.py swh/deposit/migrations/0011_auto_20180115_1510.py swh/deposit/migrations/0012_deposit_status_detail.py swh/deposit/migrations/0013_depositrequest_raw_metadata.py swh/deposit/migrations/0014_auto_20180720_1221.py swh/deposit/migrations/0015_depositrequest_typemigration.py swh/deposit/migrations/0016_auto_20190507_1408.py swh/deposit/migrations/0017_auto_20190925_0906.py +swh/deposit/migrations/0018_migrate_swhids.py swh/deposit/migrations/__init__.py swh/deposit/settings/__init__.py swh/deposit/settings/common.py swh/deposit/settings/development.py swh/deposit/settings/production.py swh/deposit/settings/testing.py swh/deposit/static/robots.txt swh/deposit/static/css/bootstrap-responsive.min.css swh/deposit/static/css/style.css swh/deposit/static/img/arrow-up-small.png swh/deposit/static/img/swh-logo-deposit.png swh/deposit/static/img/swh-logo-deposit.svg swh/deposit/static/img/icons/swh-logo-32x32.png swh/deposit/static/img/icons/swh-logo-deposit-180x180.png swh/deposit/static/img/icons/swh-logo-deposit-192x192.png swh/deposit/static/img/icons/swh-logo-deposit-270x270.png swh/deposit/templates/__init__.py swh/deposit/templates/api.html swh/deposit/templates/homepage.html swh/deposit/templates/layout.html swh/deposit/templates/deposit/__init__.py swh/deposit/templates/deposit/content.xml swh/deposit/templates/deposit/deposit_receipt.xml swh/deposit/templates/deposit/error.xml swh/deposit/templates/deposit/service_document.xml swh/deposit/templates/deposit/status.xml swh/deposit/templates/rest_framework/api.html swh/deposit/tests/__init__.py swh/deposit/tests/common.py swh/deposit/tests/conftest.py swh/deposit/tests/test_common.py swh/deposit/tests/test_gunicorn_config.py swh/deposit/tests/test_utils.py swh/deposit/tests/api/__init__.py swh/deposit/tests/api/conftest.py swh/deposit/tests/api/test_converters.py swh/deposit/tests/api/test_deposit.py swh/deposit/tests/api/test_deposit_atom.py swh/deposit/tests/api/test_deposit_binary.py swh/deposit/tests/api/test_deposit_delete.py swh/deposit/tests/api/test_deposit_list.py swh/deposit/tests/api/test_deposit_multipart.py swh/deposit/tests/api/test_deposit_private_check.py swh/deposit/tests/api/test_deposit_private_read_archive.py swh/deposit/tests/api/test_deposit_private_read_metadata.py swh/deposit/tests/api/test_deposit_private_update_status.py swh/deposit/tests/api/test_deposit_schedule.py swh/deposit/tests/api/test_deposit_status.py swh/deposit/tests/api/test_deposit_update.py swh/deposit/tests/api/test_exception.py swh/deposit/tests/api/test_parser.py swh/deposit/tests/api/test_service_document.py swh/deposit/tests/api/data/atom/codemeta-sample.xml swh/deposit/tests/api/data/atom/entry-data-badly-formatted.xml swh/deposit/tests/api/data/atom/entry-data-deposit-binary.xml swh/deposit/tests/api/data/atom/entry-data-empty-body.xml swh/deposit/tests/api/data/atom/entry-data-ko.xml swh/deposit/tests/api/data/atom/entry-data-minimal.xml swh/deposit/tests/api/data/atom/entry-data-parsing-error-prone.xml swh/deposit/tests/api/data/atom/entry-data0.xml swh/deposit/tests/api/data/atom/entry-data1.xml swh/deposit/tests/api/data/atom/entry-data2.xml swh/deposit/tests/api/data/atom/entry-data3.xml swh/deposit/tests/api/data/atom/entry-update-in-place.xml swh/deposit/tests/api/data/atom/error-with-decimal.xml swh/deposit/tests/api/data/atom/metadata.xml swh/deposit/tests/api/data/atom/tei-sample.xml swh/deposit/tests/cli/__init__.py swh/deposit/tests/cli/test_client.py swh/deposit/tests/cli/data/atom/codemeta-sample.xml swh/deposit/tests/cli/data/atom/entry-data-badly-formatted.xml swh/deposit/tests/cli/data/atom/entry-data-deposit-binary.xml swh/deposit/tests/cli/data/atom/entry-data-empty-body.xml swh/deposit/tests/cli/data/atom/entry-data-ko.xml swh/deposit/tests/cli/data/atom/entry-data-minimal.xml swh/deposit/tests/cli/data/atom/entry-data-parsing-error-prone.xml swh/deposit/tests/cli/data/atom/entry-data0.xml swh/deposit/tests/cli/data/atom/entry-data1.xml swh/deposit/tests/cli/data/atom/entry-data2.xml swh/deposit/tests/cli/data/atom/entry-data3.xml swh/deposit/tests/cli/data/atom/entry-update-in-place.xml swh/deposit/tests/cli/data/atom/error-with-decimal.xml swh/deposit/tests/cli/data/atom/metadata.xml swh/deposit/tests/cli/data/atom/tei-sample.xml swh/deposit/tests/loader/__init__.py swh/deposit/tests/loader/common.py swh/deposit/tests/loader/conftest.py swh/deposit/tests/loader/test_checker.py swh/deposit/tests/loader/test_client.py swh/deposit/tests/loader/test_tasks.py swh/deposit/tests/loader/data/http_example.org/hello.json swh/deposit/tests/loader/data/http_example.org/hello_you swh/deposit/tests/loader/data/https_deposit.softwareheritage.org/1_private_test_1_check swh/deposit/tests/loader/data/https_deposit.softwareheritage.org/1_private_test_2_check swh/deposit/tests/loader/data/https_deposit.softwareheritage.org/1_private_test_999_meta swh/deposit/tests/loader/data/https_deposit.softwareheritage.org/1_private_test_999_raw swh/deposit/tests/loader/data/https_deposit.softwareheritage.org/1_private_test_999_update swh/deposit/tests/loader/data/https_nowhere.org/1_private_test_1_check swh/deposit/tests/loader/data/https_nowhere.org/1_private_test_1_metadata swh/deposit/tests/loader/data/https_nowhere.org/1_private_test_1_raw \ No newline at end of file diff --git a/swh.deposit.egg-info/requires.txt b/swh.deposit.egg-info/requires.txt index 9898c478..afc37349 100644 --- a/swh.deposit.egg-info/requires.txt +++ b/swh.deposit.egg-info/requires.txt @@ -1,30 +1,30 @@ vcversioner click xmltodict iso8601 requests swh.core>=0.0.75 [server] Django<3 djangorestframework swh.core[http] swh.loader.core>=0.0.71 swh.scheduler>=0.0.39 -swh.model>=0.0.26 +swh.model>=0.1.0 [testing] pytest pytest-django pytest-mock swh.scheduler[testing] swh.loader.core[testing] pytest-postgresql>=2.1.0 requests_mock django-stubs Django<3 djangorestframework swh.core[http] swh.loader.core>=0.0.71 swh.scheduler>=0.0.39 -swh.model>=0.0.26 +swh.model>=0.1.0 diff --git a/swh/deposit/api/private/deposit_update_status.py b/swh/deposit/api/private/deposit_update_status.py index 87d94f70..b4e2b898 100644 --- a/swh/deposit/api/private/deposit_update_status.py +++ b/swh/deposit/api/private/deposit_update_status.py @@ -1,82 +1,111 @@ -# Copyright (C) 2017-2019 The Software Heritage developers +# Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from rest_framework.parsers import JSONParser -from swh.model.identifiers import persistent_identifier, REVISION, DIRECTORY +from swh.model.identifiers import DIRECTORY, persistent_identifier, REVISION, SNAPSHOT from . import SWHPrivateAPIView from ..common import SWHPutDepositAPI from ...errors import make_error_dict, BAD_REQUEST from ...models import Deposit, DEPOSIT_STATUS_DETAIL from ...models import DEPOSIT_STATUS_LOAD_SUCCESS +MANDATORY_KEYS = ["origin_url", "revision_id", "directory_id", "snapshot_id"] + + class SWHUpdateStatusDeposit(SWHPrivateAPIView, SWHPutDepositAPI): """Deposit request class to update the deposit's status. HTTP verbs supported: PUT """ parser_classes = (JSONParser,) def additional_checks(self, request, headers, collection_name, deposit_id=None): """Enrich existing checks to the default ones. New checks: - Ensure the status is provided - Ensure it exists + - no missing information on load success update """ data = request.data status = data.get("status") if not status: msg = "The status key is mandatory with possible values %s" % list( DEPOSIT_STATUS_DETAIL.keys() ) return make_error_dict(BAD_REQUEST, msg) if status not in DEPOSIT_STATUS_DETAIL: msg = "Possible status in %s" % list(DEPOSIT_STATUS_DETAIL.keys()) return make_error_dict(BAD_REQUEST, msg) if status == DEPOSIT_STATUS_LOAD_SUCCESS: - swh_id = data.get("revision_id") - if not swh_id: - msg = "Updating status to %s requires a revision_id key" % (status,) + missing_keys = [] + for key in MANDATORY_KEYS: + value = data.get(key) + if value is None: + missing_keys.append(key) + + if missing_keys: + msg = ( + f"Updating deposit status to {status}" + f" requires information {','.join(missing_keys)}" + ) return make_error_dict(BAD_REQUEST, msg) return {} def process_put(self, request, headers, collection_name, deposit_id): - """Update the deposit's status + """Update the deposit with status and SWHIDs Returns: 204 No content + 400 Bad request if checks fail """ - deposit = Deposit.objects.get(pk=deposit_id) - deposit.status = request.data["status"] # checks already done before + data = request.data - origin_url = request.data.get("origin_url") + deposit = Deposit.objects.get(pk=deposit_id) - dir_id = request.data.get("directory_id") - if dir_id: - deposit.swh_id = persistent_identifier(DIRECTORY, dir_id) + status = data["status"] + deposit.status = status + if status == DEPOSIT_STATUS_LOAD_SUCCESS: + origin_url = data["origin_url"] + directory_id = data["directory_id"] + revision_id = data["revision_id"] + dir_id = persistent_identifier(DIRECTORY, directory_id) + snp_id = persistent_identifier(SNAPSHOT, data["snapshot_id"]) + rev_id = persistent_identifier(REVISION, revision_id) + + deposit.swh_id = dir_id + # new id with contextual information deposit.swh_id_context = persistent_identifier( - DIRECTORY, dir_id, metadata={"origin": origin_url} + DIRECTORY, + directory_id, + metadata={ + "origin": origin_url, + "visit": snp_id, + "anchor": rev_id, + "path": "/", + }, ) - rev_id = request.data.get("revision_id") - if rev_id: - deposit.swh_anchor_id = persistent_identifier(REVISION, rev_id) + # backward compatibility for now + deposit.swh_anchor_id = rev_id deposit.swh_anchor_id_context = persistent_identifier( - REVISION, rev_id, metadata={"origin": origin_url} + REVISION, revision_id, metadata={"origin": origin_url} ) + else: # rejected + deposit.status = status deposit.save() return {} diff --git a/swh/deposit/migrations/0018_migrate_swhids.py b/swh/deposit/migrations/0018_migrate_swhids.py new file mode 100644 index 00000000..ebac5f14 --- /dev/null +++ b/swh/deposit/migrations/0018_migrate_swhids.py @@ -0,0 +1,363 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import os +import logging + +from django.db import migrations +from typing import Any, Dict, Optional, Tuple + +from swh.core import config +from swh.deposit.config import DEPOSIT_STATUS_LOAD_SUCCESS +from swh.model.hashutil import hash_to_bytes, hash_to_hex +from swh.model.identifiers import ( + parse_persistent_identifier, + persistent_identifier, + DIRECTORY, + REVISION, + SNAPSHOT, +) +from swh.storage import get_storage as get_storage_client + + +SWH_PROVIDER_URL = "https://www.softwareheritage.org" + + +logger = logging.getLogger(__name__) + + +swh_storage = None + + +def get_storage() -> Optional[Any]: + """Instantiate a storage client + + """ + settings = os.environ.get("DJANGO_SETTINGS_MODULE") + if settings != "swh.deposit.settings.production": # Bypass for now + return None + + global swh_storage + + if not swh_storage: + config_file = os.environ.get("SWH_CONFIG_FILENAME") + if not config_file: + raise ValueError( + "Production: SWH_CONFIG_FILENAME must be set to the" + " configuration file needed!" + ) + + if not os.path.exists(config_file): + raise ValueError( + "Production: configuration file %s does not exist!" % (config_file,) + ) + + conf = config.load_named_config(config_file) + if not conf: + raise ValueError( + "Production: configuration %s does not exist." % (config_file,) + ) + + storage_config = conf.get("storage") + if not storage_config: + raise ValueError( + "Production: invalid configuration; missing 'storage' config entry." + ) + + swh_storage = get_storage_client(**storage_config) + + return swh_storage + + +def get_snapshot(storage, origin: str, revision_id: str) -> Optional[str]: + """Retrieve the snapshot targeting the revision_id for the given origin. + + """ + all_visits = storage.origin_visit_get(origin) + for visit in all_visits: + if not visit["snapshot"]: + continue + detail_snapshot = storage.snapshot_get(visit["snapshot"]) + if not detail_snapshot: + continue + for branch_name, branch in detail_snapshot["branches"].items(): + if branch["target_type"] == "revision": + revision = branch["target"] + if hash_to_hex(revision) == revision_id: + # Found the snapshot + return hash_to_hex(visit["snapshot"]) + return None + + +def migrate_deposit_swhid_context_not_null(apps, schema_editor): + """Migrate deposit SWHIDs to the new format. + + Migrate deposit SWHIDs to the new format. Only deposit with status done and + swh_id_context not null are concerned. + + """ + storage = get_storage() + if not storage: + logging.warning("Nothing to do") + return None + + Deposit = apps.get_model("deposit", "Deposit") + for deposit in Deposit.objects.filter( + status=DEPOSIT_STATUS_LOAD_SUCCESS, swh_id_context__isnull=False + ): + obj_dir = parse_persistent_identifier(deposit.swh_id_context) + assert obj_dir.object_type == DIRECTORY + + obj_rev = parse_persistent_identifier(deposit.swh_anchor_id) + assert obj_rev.object_type == REVISION + + if set(obj_dir.metadata.keys()) != {"origin"}: + # Assuming the migration is already done for that deposit + logger.warning( + "Deposit id %s: Migration already done, skipping", deposit.id + ) + continue + + # Starting migration + + dir_id = obj_dir.object_id + origin = obj_dir.metadata["origin"] + + check_origin = storage.origin_get({"url": origin}) + if not check_origin: + logger.warning("Deposit id %s: Origin %s not found!", deposit.id, origin) + continue + + rev_id = obj_rev.object_id + # Find the snapshot targeting the revision + snp_id = get_snapshot(storage, origin, rev_id) + if not snp_id: + logger.warning( + "Deposit id %s: Snapshot targeting revision %s not found!", + deposit.id, + rev_id, + ) + continue + + # Reference the old values to do some checks later + old_swh_id = deposit.swh_id + old_swh_id_context = deposit.swh_id_context + old_swh_anchor_id = deposit.swh_anchor_id + old_swh_anchor_id_context = deposit.swh_anchor_id_context + + # Update + deposit.swh_id_context = persistent_identifier( + DIRECTORY, + dir_id, + metadata={ + "origin": origin, + "visit": persistent_identifier(SNAPSHOT, snp_id), + "anchor": persistent_identifier(REVISION, rev_id), + "path": "/", + }, + ) + + # Ensure only deposit.swh_id_context changed + logging.debug("deposit.id: {deposit.id}") + logging.debug("deposit.swh_id: %s -> %s", old_swh_id, deposit.swh_id) + assert old_swh_id == deposit.swh_id + logging.debug( + "deposit.swh_id_context: %s -> %s", + old_swh_id_context, + deposit.swh_id_context, + ) + assert old_swh_id_context != deposit.swh_id_context + logging.debug( + "deposit.swh_anchor_id: %s -> %s", old_swh_anchor_id, deposit.swh_anchor_id + ) + assert old_swh_anchor_id == deposit.swh_anchor_id + logging.debug( + "deposit.swh_anchor_id_context: %s -> %s", + old_swh_anchor_id_context, + deposit.swh_anchor_id_context, + ) + assert old_swh_anchor_id_context == deposit.swh_anchor_id_context + + # Commit + deposit.save() + + +def resolve_origin(deposit_id: int, provider_url: str, external_id: str) -> str: + """Resolve the origin from provider-url and external-id + + For some edge case, only the external_id is used as there is some old inconsistency + from testing which exists. + + """ + map_edge_case_origin: Dict[Tuple[int, str], str] = { + ( + 76, + "hal-01588782", + ): "https://inria.halpreprod.archives-ouvertes.fr/hal-01588782", + ( + 87, + "hal-01588927", + ): "https://inria.halpreprod.archives-ouvertes.fr/hal-01588927", + (89, "hal-01588935"): "https://hal-preprod.archives-ouvertes.fr/hal-01588935", + ( + 88, + "hal-01588928", + ): "https://inria.halpreprod.archives-ouvertes.fr/hal-01588928", + ( + 90, + "hal-01588942", + ): "https://inria.halpreprod.archives-ouvertes.fr/hal-01588942", + (143, "hal-01592430"): "https://hal-preprod.archives-ouvertes.fr/hal-01592430", + ( + 75, + "hal-01588781", + ): "https://inria.halpreprod.archives-ouvertes.fr/hal-01588781", + } + origin = map_edge_case_origin.get((deposit_id, external_id)) + if origin: + return origin + + # Some simpler origin edge cases (mostly around the initial deposits) + map_origin = { + ( + SWH_PROVIDER_URL, + "je-suis-gpl", + ): "https://forge.softwareheritage.org/source/jesuisgpl/", + ( + SWH_PROVIDER_URL, + "external-id", + ): "https://hal.archives-ouvertes.fr/external-id", + } + key = (provider_url, external_id) + return map_origin.get(key, f"{provider_url.rstrip('/')}/{external_id}") + + +def migrate_deposit_swhid_context_null(apps, schema_editor): + """Migrate deposit SWHIDs to the new format. + + Migrate deposit whose swh_id_context is not set (initial deposits not migrated at + the time). Only deposit with status done and swh_id_context null are concerned. + + Note: Those deposits have their swh_id being the SWHPIDs of the revision! So we can + align them as well. + + """ + storage = get_storage() + if not storage: + logging.warning("Nothing to do") + return None + Deposit = apps.get_model("deposit", "Deposit") + for deposit in Deposit.objects.filter( + status=DEPOSIT_STATUS_LOAD_SUCCESS, swh_id_context__isnull=True + ): + obj_rev = parse_persistent_identifier(deposit.swh_id) + if obj_rev.object_type == DIRECTORY: + # Assuming the migration is already done for that deposit + logger.warning( + "Deposit id %s: Migration already done, skipping", deposit.id + ) + continue + + # Ensuring Migration not done + assert obj_rev.object_type == REVISION + + assert deposit.swh_id is not None + assert deposit.swh_id_context is None + assert deposit.swh_anchor_id is None + assert deposit.swh_anchor_id_context is None + + rev_id = obj_rev.object_id + revisions = list(storage.revision_get([hash_to_bytes(rev_id)])) + if not revisions: + logger.warning("Deposit id %s: Revision %s not found!", deposit.id, rev_id) + continue + revision = revisions[0] + + provider_url = deposit.client.provider_url + external_id = deposit.external_id + + origin = resolve_origin(deposit.id, provider_url, external_id) + check_origin = storage.origin_get({"url": origin}) + if not check_origin: + logger.warning("Deposit id %s: Origin %s not found!", deposit.id, origin) + continue + + dir_id = hash_to_hex(revision["directory"]) + + # Reference the old values to do some checks later + old_swh_id = deposit.swh_id + old_swh_id_context = deposit.swh_id_context + old_swh_anchor_id = deposit.swh_anchor_id + old_swh_anchor_id_context = deposit.swh_anchor_id_context + + # retrieve the snapshot from the archive + snp_id = get_snapshot(storage, origin, rev_id) + if not snp_id: + logger.warning( + "Deposit id %s: Snapshot targeting revision %s not found!", + deposit.id, + rev_id, + ) + continue + + # New SWHIDs ids + deposit.swh_id = persistent_identifier(DIRECTORY, dir_id) + deposit.swh_id_context = persistent_identifier( + DIRECTORY, + dir_id, + metadata={ + "origin": origin, + "visit": persistent_identifier(SNAPSHOT, snp_id), + "anchor": persistent_identifier(REVISION, rev_id), + "path": "/", + }, + ) + # Realign the remaining deposit SWHIDs fields + deposit.swh_anchor_id = persistent_identifier(REVISION, rev_id) + deposit.swh_anchor_id_context = persistent_identifier( + REVISION, rev_id, metadata={"origin": origin,} + ) + + # Ensure only deposit.swh_id_context changed + logging.debug("deposit.id: {deposit.id}") + logging.debug("deposit.swh_id: %s -> %s", old_swh_id, deposit.swh_id) + + assert old_swh_id != deposit.swh_id + logging.debug( + "deposit.swh_id_context: %s -> %s", + old_swh_id_context, + deposit.swh_id_context, + ) + assert old_swh_id_context != deposit.swh_id_context + assert deposit.swh_id_context is not None + logging.debug( + "deposit.swh_anchor_id: %s -> %s", old_swh_anchor_id, deposit.swh_anchor_id + ) + assert deposit.swh_anchor_id == old_swh_id + assert deposit.swh_anchor_id is not None + logging.debug( + "deposit.swh_anchor_id_context: %s -> %s", + old_swh_anchor_id_context, + deposit.swh_anchor_id_context, + ) + assert deposit.swh_anchor_id_context is not None + + deposit.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("deposit", "0017_auto_20190925_0906"), + ] + + operations = [ + # Migrate and make the operations possibly reversible + # https://docs.djangoproject.com/en/3.0/ref/migration-operations/#django.db.migrations.operations.RunPython.noop # noqa + migrations.RunPython( + migrate_deposit_swhid_context_not_null, + reverse_code=migrations.RunPython.noop, + ), + migrations.RunPython( + migrate_deposit_swhid_context_null, reverse_code=migrations.RunPython.noop + ), + ] diff --git a/swh/deposit/settings/production.py b/swh/deposit/settings/production.py index 725536b0..5cc7c8b1 100644 --- a/swh/deposit/settings/production.py +++ b/swh/deposit/settings/production.py @@ -1,110 +1,110 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os from .common import * # noqa from .common import ALLOWED_HOSTS from swh.core import config ALLOWED_HOSTS += ["deposit.softwareheritage.org"] # Setup support for proxy headers USE_X_FORWARDED_HOST = True SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") DEBUG = False # Database # https://docs.djangoproject.com/en/1.10/ref/settings/#databases # https://docs.djangoproject.com/en/1.10/ref/settings/#std:setting-DATABASES # https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/#databases # Retrieve the deposit's configuration file # and check the required setup is ok # If not raise an error explaining the errors config_file = os.environ.get("SWH_CONFIG_FILENAME") if not config_file: raise ValueError( - "Production: SWH_CONFIG_FILENANE must be set to the" + "Production: SWH_CONFIG_FILENAME must be set to the" " configuration file needed!" ) if not os.path.exists(config_file): raise ValueError( "Production: configuration file %s does not exist!" % (config_file,) ) conf = config.load_named_config(config_file) if not conf: raise ValueError("Production: configuration %s does not exist." % (config_file,)) for key in ("scheduler", "private"): if not conf.get(key): raise ValueError( "Production: invalid configuration; missing %s config entry." % (key,) ) ALLOWED_HOSTS += conf.get("allowed_hosts", []) private_conf = conf["private"] SECRET_KEY = private_conf["secret_key"] # https://docs.djangoproject.com/en/1.10/ref/settings/#logging LOGGING = { "version": 1, "disable_existing_loggers": False, "formatters": { "standard": { "format": "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", # noqa "datefmt": "%d/%b/%Y %H:%M:%S", }, }, "handlers": { "console": { "level": "INFO", "class": "logging.StreamHandler", "formatter": "standard", }, }, "loggers": { "django": {"handlers": ["console"], "level": "INFO", "propagate": True,}, }, } # database db_conf = private_conf.get("db", {"name": "unset"}) db = { "ENGINE": "django.db.backends.postgresql", "NAME": db_conf["name"], } db_user = db_conf.get("user") if db_user: db["USER"] = db_user db_pass = db_conf.get("password") if db_pass: db["PASSWORD"] = db_pass db_host = db_conf.get("host") if db_host: db["HOST"] = db_host db_port = db_conf.get("port") if db_port: db["PORT"] = db_port # https://docs.djangoproject.com/en/1.10/ref/settings/#databases DATABASES = { "default": db, } # Upload user directory # https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-MEDIA_ROOT MEDIA_ROOT = private_conf.get("media_root") diff --git a/swh/deposit/tests/api/test_deposit_private_update_status.py b/swh/deposit/tests/api/test_deposit_private_update_status.py index c9bc27a5..fa05bf48 100644 --- a/swh/deposit/tests/api/test_deposit_private_update_status.py +++ b/swh/deposit/tests/api/test_deposit_private_update_status.py @@ -1,154 +1,200 @@ -# Copyright (C) 2017-2019 The Software Heritage developers +# Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import copy import json from django.urls import reverse from rest_framework import status -from swh.deposit.models import Deposit, DEPOSIT_STATUS_DETAIL +from swh.model.identifiers import DIRECTORY, persistent_identifier, REVISION, SNAPSHOT + +from swh.deposit.api.private.deposit_update_status import MANDATORY_KEYS + +from swh.deposit.models import Deposit from swh.deposit.config import ( PRIVATE_PUT_DEPOSIT, - DEPOSIT_STATUS_VERIFIED, DEPOSIT_STATUS_LOAD_SUCCESS, + DEPOSIT_STATUS_LOAD_FAILURE, ) PRIVATE_PUT_DEPOSIT_NC = PRIVATE_PUT_DEPOSIT + "-nc" def private_check_url_endpoints(collection, deposit): """There are 2 endpoints to check (one with collection, one without)""" return [ reverse(PRIVATE_PUT_DEPOSIT, args=[collection.name, deposit.id]), reverse(PRIVATE_PUT_DEPOSIT_NC, args=[deposit.id]), ] -def test_update_deposit_status( +def test_update_deposit_status_success_with_info( authenticated_client, deposit_collection, ready_deposit_verified ): - """Existing status for update should return a 204 response + """Update deposit with load success should require all information to succeed """ deposit = ready_deposit_verified + expected_status = DEPOSIT_STATUS_LOAD_SUCCESS + origin_url = "something" + directory_id = "42a13fc721c8716ff695d0d62fc851d641f3a12b" + revision_id = "47dc6b4636c7f6cba0df83e3d5490bf4334d987e" + snapshot_id = "68c0d26104d47e278dd6be07ed61fafb561d0d20" + + full_body_info = { + "status": DEPOSIT_STATUS_LOAD_SUCCESS, + "revision_id": revision_id, + "directory_id": directory_id, + "snapshot_id": snapshot_id, + "origin_url": origin_url, + } for url in private_check_url_endpoints(deposit_collection, deposit): - possible_status = set(DEPOSIT_STATUS_DETAIL.keys()) - set( - [DEPOSIT_STATUS_LOAD_SUCCESS] + dir_id = persistent_identifier(DIRECTORY, directory_id) + rev_id = persistent_identifier(REVISION, revision_id) + snp_id = persistent_identifier(SNAPSHOT, snapshot_id) + + expected_swh_id = "swh:1:dir:%s" % directory_id + expected_swh_id_context = ( + f"{dir_id};origin={origin_url};" + f"visit={snp_id};anchor={rev_id};path=/" ) + expected_swh_anchor_id = rev_id + expected_swh_anchor_id_context = f"{rev_id};origin={origin_url}" - for _status in possible_status: - response = authenticated_client.put( - url, - content_type="application/json", - data=json.dumps({"status": _status}), - ) + response = authenticated_client.put( + url, content_type="application/json", data=json.dumps(full_body_info), + ) - assert response.status_code == status.HTTP_204_NO_CONTENT + assert response.status_code == status.HTTP_204_NO_CONTENT - deposit = Deposit.objects.get(pk=deposit.id) - assert deposit.status == _status + deposit = Deposit.objects.get(pk=deposit.id) + assert deposit.status == expected_status + assert deposit.swh_id == expected_swh_id + assert deposit.swh_id_context == expected_swh_id_context + assert deposit.swh_anchor_id == expected_swh_anchor_id + assert deposit.swh_anchor_id_context == expected_swh_anchor_id_context - deposit.status = DEPOSIT_STATUS_VERIFIED - deposit.save() # hack the same deposit + # Reset deposit + deposit = ready_deposit_verified + deposit.save() -def test_update_deposit_status_with_info( +def test_update_deposit_status_rejected_with_info( authenticated_client, deposit_collection, ready_deposit_verified ): - """Existing status for update with info should return a 204 response + """Update deposit with rejected status needs few information to succeed """ deposit = ready_deposit_verified - for url in private_check_url_endpoints(deposit_collection, deposit): - expected_status = DEPOSIT_STATUS_LOAD_SUCCESS - origin_url = "something" - directory_id = "42a13fc721c8716ff695d0d62fc851d641f3a12b" - revision_id = "47dc6b4636c7f6cba0df83e3d5490bf4334d987e" - expected_swh_id = "swh:1:dir:%s" % directory_id - expected_swh_id_context = "swh:1:dir:%s;origin=%s" % (directory_id, origin_url) - expected_swh_anchor_id = "swh:1:rev:%s" % revision_id - expected_swh_anchor_id_context = "swh:1:rev:%s;origin=%s" % ( - revision_id, - origin_url, - ) + for url in private_check_url_endpoints(deposit_collection, deposit): response = authenticated_client.put( url, content_type="application/json", - data=json.dumps( - { - "status": expected_status, - "revision_id": revision_id, - "directory_id": directory_id, - "origin_url": origin_url, - } - ), + data=json.dumps({"status": DEPOSIT_STATUS_LOAD_FAILURE}), ) assert response.status_code == status.HTTP_204_NO_CONTENT deposit = Deposit.objects.get(pk=deposit.id) - assert deposit.status == expected_status - assert deposit.swh_id == expected_swh_id - assert deposit.swh_id_context == expected_swh_id_context - assert deposit.swh_anchor_id == expected_swh_anchor_id - assert deposit.swh_anchor_id_context == expected_swh_anchor_id_context + assert deposit.status == DEPOSIT_STATUS_LOAD_FAILURE + + assert deposit.swh_id is None + assert deposit.swh_id_context is None + assert deposit.swh_anchor_id is None + assert deposit.swh_anchor_id_context is None - deposit.swh_id = None - deposit.swh_id_context = None - deposit.swh_anchor_id = None - deposit.swh_anchor_id_context = None - deposit.status = DEPOSIT_STATUS_VERIFIED + # Reset status + deposit = ready_deposit_verified deposit.save() +def test_update_deposit_status_success_with_incomplete_data( + authenticated_client, deposit_collection, ready_deposit_verified +): + """Update deposit status with status success and incomplete information should fail + + """ + deposit = ready_deposit_verified + + origin_url = "something" + directory_id = "42a13fc721c8716ff695d0d62fc851d641f3a12b" + revision_id = "47dc6b4636c7f6cba0df83e3d5490bf4334d987e" + snapshot_id = "68c0d26104d47e278dd6be07ed61fafb561d0d20" + + new_status = DEPOSIT_STATUS_LOAD_SUCCESS + full_body_info = { + "status": new_status, + "revision_id": revision_id, + "directory_id": directory_id, + "snapshot_id": snapshot_id, + "origin_url": origin_url, + } + + for url in private_check_url_endpoints(deposit_collection, deposit): + for key in MANDATORY_KEYS: + # Crafting body with missing information so that it raises + body = copy.deepcopy(full_body_info) + body.pop(key) # make the body incomplete + + response = authenticated_client.put( + url, content_type="application/json", data=json.dumps(body), + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert ( + f"deposit status to {new_status} requires information {key}" + in response.content.decode("utf-8") + ) + + def test_update_deposit_status_will_fail_with_unknown_status( authenticated_client, deposit_collection, ready_deposit_verified ): """Unknown status for update should return a 400 response """ deposit = ready_deposit_verified for url in private_check_url_endpoints(deposit_collection, deposit): response = authenticated_client.put( url, content_type="application/json", data=json.dumps({"status": "unknown"}) ) assert response.status_code == status.HTTP_400_BAD_REQUEST def test_update_deposit_status_will_fail_with_no_status_key( authenticated_client, deposit_collection, ready_deposit_verified ): """No status provided for update should return a 400 response """ deposit = ready_deposit_verified for url in private_check_url_endpoints(deposit_collection, deposit): response = authenticated_client.put( url, content_type="application/json", data=json.dumps({"something": "something"}), ) assert response.status_code == status.HTTP_400_BAD_REQUEST def test_update_deposit_status_success_without_swh_id_fail( authenticated_client, deposit_collection, ready_deposit_verified ): """Providing successful status without swh_id should return a 400 """ deposit = ready_deposit_verified for url in private_check_url_endpoints(deposit_collection, deposit): response = authenticated_client.put( url, content_type="application/json", data=json.dumps({"status": DEPOSIT_STATUS_LOAD_SUCCESS}), ) assert response.status_code == status.HTTP_400_BAD_REQUEST diff --git a/swh/deposit/tests/cli/test_client.py b/swh/deposit/tests/cli/test_client.py index 3cfcc2b2..7b5fd7ff 100644 --- a/swh/deposit/tests/cli/test_client.py +++ b/swh/deposit/tests/cli/test_client.py @@ -1,457 +1,457 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import contextlib import logging import os import re from unittest.mock import MagicMock from click.testing import CliRunner import pytest from swh.deposit.client import PublicApiDepositClient, MaintenanceError from swh.deposit.cli.client import generate_slug, _url, _client, _collection, InputError from swh.deposit.cli import deposit as cli from ..conftest import TEST_USER EXAMPLE_SERVICE_DOCUMENT = { "service": {"workspace": {"collection": {"sword:name": "softcol",}}} } @pytest.fixture def slug(): return generate_slug() @pytest.fixture def client_mock(mocker, slug): - """A succesfull deposit client with hard-coded default values + """A successful deposit client with hard-coded default values """ mocker.patch("swh.deposit.cli.client.generate_slug", return_value=slug) mock_client = MagicMock() mocker.patch("swh.deposit.cli.client._client", return_value=mock_client) mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT mock_client.deposit_create.return_value = '{"foo": "bar"}' return mock_client @pytest.fixture def client_mock_api_down(mocker, slug): """A mock client whose connection with api fails due to maintenance issue """ mocker.patch("swh.deposit.cli.client.generate_slug", return_value=slug) mock_client = MagicMock() mocker.patch("swh.deposit.cli.client._client", return_value=mock_client) mock_client.service_document.side_effect = MaintenanceError( "Database backend maintenance: Temporarily unavailable, try again later." ) return mock_client def test_url(): assert _url("http://deposit") == "http://deposit/1" assert _url("https://other/1") == "https://other/1" def test_client(): client = _client("http://deposit", "user", "pass") assert isinstance(client, PublicApiDepositClient) def test_collection_error(): mock_client = MagicMock() mock_client.service_document.return_value = {"error": "something went wrong"} with pytest.raises(InputError) as e: _collection(mock_client) assert "Service document retrieval: something went wrong" == str(e.value) def test_collection_ok(): mock_client = MagicMock() mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT collection_name = _collection(mock_client) assert collection_name == "softcol" def test_collection_ko_because_downtime(): mock_client = MagicMock() mock_client.service_document.side_effect = MaintenanceError("downtime") with pytest.raises(MaintenanceError, match="downtime"): _collection(mock_client) def test_deposit_with_server_down_for_maintenance( sample_archive, mocker, caplog, client_mock_api_down, slug, tmp_path ): """ Deposit failure due to maintenance down time should be explicit """ runner = CliRunner() result = runner.invoke( cli, [ "upload", "--url", "mock://deposit.swh/1", "--username", TEST_USER["username"], "--password", TEST_USER["password"], "--name", "test-project", "--archive", sample_archive["path"], "--author", "Jane Doe", ], ) assert result.exit_code == 1, result.output assert result.output == "" assert caplog.record_tuples == [ ( "swh.deposit.cli.client", logging.ERROR, "Database backend maintenance: Temporarily unavailable, try again later.", ) ] client_mock_api_down.service_document.assert_called_once_with() def test_single_minimal_deposit( sample_archive, mocker, caplog, client_mock, slug, tmp_path ): """ from: https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit """ # noqa metadata_path = os.path.join(tmp_path, "metadata.xml") mocker.patch( "swh.deposit.cli.client.tempfile.TemporaryDirectory", return_value=contextlib.nullcontext(str(tmp_path)), ) runner = CliRunner() result = runner.invoke( cli, [ "upload", "--url", "mock://deposit.swh/1", "--username", TEST_USER["username"], "--password", TEST_USER["password"], "--name", "test-project", "--archive", sample_archive["path"], "--author", "Jane Doe", ], ) assert result.exit_code == 0, result.output assert result.output == "" assert caplog.record_tuples == [ ("swh.deposit.cli.client", logging.INFO, '{"foo": "bar"}'), ] client_mock.deposit_create.assert_called_once_with( archive=sample_archive["path"], collection="softcol", in_progress=False, metadata=metadata_path, slug=slug, ) with open(metadata_path) as fd: assert ( fd.read() == f"""\ \ttest-project \t{slug} \t \t\tJane Doe \t """ ) def test_metadata_validation(sample_archive, mocker, caplog, tmp_path): """ from: https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit """ # noqa slug = generate_slug() mocker.patch("swh.deposit.cli.client.generate_slug", return_value=slug) mock_client = MagicMock() mocker.patch("swh.deposit.cli.client._client", return_value=mock_client) mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT mock_client.deposit_create.return_value = '{"foo": "bar"}' metadata_path = os.path.join(tmp_path, "metadata.xml") mocker.patch( "swh.deposit.cli.client.tempfile.TemporaryDirectory", return_value=contextlib.nullcontext(str(tmp_path)), ) with open(metadata_path, "a"): pass # creates the file runner = CliRunner() # Test missing author result = runner.invoke( cli, [ "upload", "--url", "mock://deposit.swh/1", "--username", TEST_USER["username"], "--password", TEST_USER["password"], "--name", "test-project", "--archive", sample_archive["path"], ], ) assert result.exit_code == 1, result.output assert result.output == "" assert len(caplog.record_tuples) == 1 (_logger, level, message) = caplog.record_tuples[0] assert level == logging.ERROR assert " --author " in message # Clear mocking state caplog.clear() mock_client.reset_mock() # Test missing name result = runner.invoke( cli, [ "upload", "--url", "mock://deposit.swh/1", "--username", TEST_USER["username"], "--password", TEST_USER["password"], "--archive", sample_archive["path"], "--author", "Jane Doe", ], ) assert result.exit_code == 1, result.output assert result.output == "" assert len(caplog.record_tuples) == 1 (_logger, level, message) = caplog.record_tuples[0] assert level == logging.ERROR assert " --name " in message # Clear mocking state caplog.clear() mock_client.reset_mock() # Test both --metadata and --author result = runner.invoke( cli, [ "upload", "--url", "mock://deposit.swh/1", "--username", TEST_USER["username"], "--password", TEST_USER["password"], "--archive", sample_archive["path"], "--metadata", metadata_path, "--author", "Jane Doe", ], ) assert result.exit_code == 1, result.output assert result.output == "" assert len(caplog.record_tuples) == 1 (_logger, level, message) = caplog.record_tuples[0] assert level == logging.ERROR assert re.search("--metadata.*is incompatible with", message) # Clear mocking state caplog.clear() mock_client.reset_mock() def test_single_deposit_slug_generation( sample_archive, mocker, caplog, tmp_path, client_mock ): """ from: https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit """ # noqa slug = "my-slug" collection = "my-collection" metadata_path = os.path.join(tmp_path, "metadata.xml") mocker.patch( "swh.deposit.cli.client.tempfile.TemporaryDirectory", return_value=contextlib.nullcontext(str(tmp_path)), ) runner = CliRunner() result = runner.invoke( cli, [ "upload", "--url", "mock://deposit.swh/1", "--username", TEST_USER["username"], "--password", TEST_USER["password"], "--name", "test-project", "--archive", sample_archive["path"], "--slug", slug, "--collection", collection, "--author", "Jane Doe", ], ) assert result.exit_code == 0, result.output assert result.output == "" assert caplog.record_tuples == [ ("swh.deposit.cli.client", logging.INFO, '{"foo": "bar"}'), ] client_mock.deposit_create.assert_called_once_with( archive=sample_archive["path"], collection=collection, in_progress=False, metadata=metadata_path, slug=slug, ) with open(metadata_path) as fd: assert ( fd.read() == """\ \ttest-project \tmy-slug \t \t\tJane Doe \t """ ) def test_multisteps_deposit( sample_archive, atom_dataset, mocker, caplog, datadir, client_mock, slug ): """ from: https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#multisteps-deposit """ # noqa slug = generate_slug() mocker.patch("swh.deposit.cli.client.generate_slug", return_value=slug) # https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#create-an-incomplete-deposit client_mock.deposit_create.return_value = '{"deposit_id": "42"}' runner = CliRunner() result = runner.invoke( cli, [ "upload", "--url", "mock://deposit.swh/1", "--username", TEST_USER["username"], "--password", TEST_USER["password"], "--archive", sample_archive["path"], "--partial", ], ) assert result.exit_code == 0, result.output assert result.output == "" assert caplog.record_tuples == [ ("swh.deposit.cli.client", logging.INFO, '{"deposit_id": "42"}'), ] client_mock.deposit_create.assert_called_once_with( archive=sample_archive["path"], collection="softcol", in_progress=True, metadata=None, slug=slug, ) # Clear mocking state caplog.clear() client_mock.reset_mock() # https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#add-content-or-metadata-to-the-deposit metadata_path = os.path.join(datadir, "atom", "entry-data-deposit-binary.xml") result = runner.invoke( cli, [ "upload", "--url", "mock://deposit.swh/1", "--username", TEST_USER["username"], "--password", TEST_USER["password"], "--metadata", metadata_path, ], ) assert result.exit_code == 0, result.output assert result.output == "" assert caplog.record_tuples == [ ("swh.deposit.cli.client", logging.INFO, '{"deposit_id": "42"}'), ] client_mock.deposit_create.assert_called_once_with( archive=None, collection="softcol", in_progress=False, metadata=metadata_path, slug=slug, ) # Clear mocking state caplog.clear() client_mock.reset_mock() diff --git a/tox.ini b/tox.ini index b4346c5c..00c7376d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,44 +1,44 @@ [tox] envlist=flake8,mypy,py3-django2 [testenv] extras = testing deps = # the dependency below is needed for now as a workaround for # https://github.com/pypa/pip/issues/6239 swh.core[http] >= 0.0.75 - dev: ipdb + dev: pdbpp pytest-cov django2: Django>=2,<3 commands = pytest \ !dev: --cov {envsitepackagesdir}/swh/deposit --cov-branch \ {envsitepackagesdir}/swh/deposit \ {posargs} [testenv:black] skip_install = true deps = black commands = {envpython} -m black --check swh [testenv:flake8] skip_install = true deps = flake8 commands = {envpython} -m flake8 \ --exclude=.tox,.git,__pycache__,.tox,.eggs,*.egg,swh/deposit/migrations [testenv:mypy] setenv = DJANGO_SETTINGS_MODULE=swh.deposit.settings.testing extras = testing deps = mypy django-stubs djangorestframework-stubs commands = mypy swh diff --git a/version.txt b/version.txt index 46ca80c7..97436e93 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.86-0-g85e1ff3e \ No newline at end of file +v0.0.87-0-ga631dabb \ No newline at end of file