diff --git a/swh/deposit/api/common.py b/swh/deposit/api/common.py
index 733996c5..dc49772c 100644
--- a/swh/deposit/api/common.py
+++ b/swh/deposit/api/common.py
@@ -1,826 +1,828 @@
# Copyright (C) 2017 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import hashlib
from abc import ABCMeta, abstractmethod
from django.core.urlresolvers import reverse
from django.http import HttpResponse
from django.shortcuts import render
from django.utils import timezone
from rest_framework import status
from rest_framework.views import APIView
from rest_framework.permissions import IsAuthenticated, AllowAny
from ..config import SWHDefaultConfig, EDIT_SE_IRI, EM_IRI, CONT_FILE_IRI
from ..config import ARCHIVE_KEY, METADATA_KEY
from ..models import Deposit, DepositRequest, DepositCollection
from ..models import DepositRequestType, DepositClient
from ..parsers import parse_xml
from ..errors import MAX_UPLOAD_SIZE_EXCEEDED, BAD_REQUEST, ERROR_CONTENT
from ..errors import CHECKSUM_MISMATCH, make_error_dict, MEDIATION_NOT_ALLOWED
from ..errors import make_error_response_from_dict, FORBIDDEN
from ..errors import NOT_FOUND, make_error_response, METHOD_NOT_ALLOWED
ACCEPT_PACKAGINGS = ['http://purl.org/net/sword/package/SimpleZip']
ACCEPT_CONTENT_TYPES = ['application/zip']
class SWHAPIView(APIView):
"""Mixin intended as a based API view to enforce the basic
authentication check
"""
permission_classes = (IsAuthenticated, )
class SWHPrivateAPIView(SWHAPIView):
"""Mixin intended as private api (so no authentication) based API view
(for the private ones).
"""
permission_classes = (AllowAny, )
class SWHBaseDeposit(SWHDefaultConfig, SWHAPIView, metaclass=ABCMeta):
"""Base deposit request class sharing multiple common behaviors.
"""
def __init__(self):
super().__init__()
deposit_request_types = DepositRequestType.objects.all()
self.deposit_request_types = {
type.name: type for type in deposit_request_types
}
def _read_headers(self, req):
"""Read and unify the necessary headers from the request (those are
not stored in the same location or not properly formatted).
Args:
req (Request): Input request
Returns:
Dictionary with the following keys (some associated values may be
None):
- content-type
- content-length
- in-progress
- content-disposition
- packaging
- slug
- on-behalf-of
"""
meta = req._request.META
content_type = req.content_type
content_length = meta.get('CONTENT_LENGTH')
if content_length and isinstance(content_length, str):
content_length = int(content_length)
# final deposit if not provided
in_progress = meta.get('HTTP_IN_PROGRESS', False)
content_disposition = meta.get('HTTP_CONTENT_DISPOSITION')
if isinstance(in_progress, str):
in_progress = in_progress.lower() == 'true'
content_md5sum = meta.get('HTTP_CONTENT_MD5')
if content_md5sum:
content_md5sum = bytes.fromhex(content_md5sum)
packaging = meta.get('HTTP_PACKAGING')
slug = meta.get('HTTP_SLUG')
on_behalf_of = meta.get('HTTP_ON_BEHALF_OF')
metadata_relevant = meta.get('HTTP_METADATA_RELEVANT')
return {
'content-type': content_type,
'content-length': content_length,
'in-progress': in_progress,
'content-disposition': content_disposition,
'content-md5sum': content_md5sum,
'packaging': packaging,
'slug': slug,
'on-behalf-of': on_behalf_of,
'metadata-relevant': metadata_relevant,
}
def _compute_md5(self, filehandler):
"""Compute uploaded file's md5 sum.
Args:
filehandler (InMemoryUploadedFile): the file to compute the md5
hash
Returns:
the md5 checksum (str)
"""
h = hashlib.md5()
for chunk in filehandler:
h.update(chunk)
return h.digest()
def _deposit_put(self, deposit_id=None, in_progress=False,
external_id=None):
"""Save/Update a deposit in db.
Args:
deposit_id (int): deposit identifier
in_progress (dict): The deposit's status
external_id (str): The external identifier to associate to
the deposit
Returns:
The Deposit instance saved or updated.
"""
if in_progress is False:
complete_date = timezone.now()
status_type = 'ready'
else:
complete_date = None
status_type = 'partial'
if not deposit_id:
deposit = Deposit(collection=self._collection,
external_id=external_id,
complete_date=complete_date,
status=status_type,
client=self._client)
else:
deposit = Deposit.objects.get(pk=deposit_id)
# update metadata
deposit.complete_date = complete_date
deposit.status = status_type
deposit.save()
return deposit
def _deposit_request_put(self, deposit, deposit_request_data,
replace_metadata=False, replace_archives=False):
"""Save a deposit request with metadata attached to a deposit.
Args:
deposit (Deposit): The deposit concerned by the request
deposit_request_data (dict): The dictionary with at most 2 deposit
request types (archive, metadata) to associate to the deposit
replace_metadata (bool): Flag defining if we add or update
existing metadata to the deposit
replace_archives (bool): Flag defining if we add or update
archives to existing deposit
Returns:
None
"""
if replace_metadata:
DepositRequest.objects.filter(
deposit=deposit,
type=self.deposit_request_types[METADATA_KEY]).delete()
if replace_archives:
DepositRequest.objects.filter(
deposit=deposit,
type=self.deposit_request_types[ARCHIVE_KEY]).delete()
deposit_request = None
archive_file = deposit_request_data.get(ARCHIVE_KEY)
if archive_file:
deposit_request = DepositRequest(
type=self.deposit_request_types[ARCHIVE_KEY],
deposit=deposit,
archive=archive_file)
deposit_request.save()
metadata = deposit_request_data.get(METADATA_KEY)
if metadata:
deposit_request = DepositRequest(
type=self.deposit_request_types[METADATA_KEY],
deposit=deposit,
metadata=metadata)
deposit_request.save()
assert deposit_request is not None
def _delete_archives(self, collection_name, deposit_id):
"""Delete archives reference from the deposit id.
"""
try:
deposit = Deposit.objects.get(pk=deposit_id)
except Deposit.DoesNotExist:
return make_error_dict(
NOT_FOUND,
'The deposit %s does not exist' % deposit_id)
DepositRequest.objects.filter(
deposit=deposit,
type=self.deposit_request_types[ARCHIVE_KEY]).delete()
return {}
def _delete_deposit(self, collection_name, deposit_id):
"""Delete deposit reference.
Args:
collection_name (str): Client's name
deposit_id (id): The deposit to delete
Returns
Empty dict when ok.
Dict with error key to describe the failure.
"""
try:
deposit = Deposit.objects.get(pk=deposit_id)
except Deposit.DoesNotExist:
return make_error_dict(
NOT_FOUND,
'The deposit %s does not exist' % deposit_id)
if deposit.collection.name != collection_name:
summary = 'Cannot delete a deposit from another collection'
description = "Deposit %s does not belong to the collection %s" % (
deposit_id, collection_name)
return make_error_dict(
BAD_REQUEST,
summary=summary,
verbose_description=description)
DepositRequest.objects.filter(deposit=deposit).delete()
deposit.delete()
return {}
def _check_preconditions_on(self, filehandler, md5sum,
content_length=None):
"""Check preconditions on provided file are respected. That is the
length and/or the md5sum hash match the file's content.
Args:
filehandler (InMemoryUploadedFile): The file to check
md5sum (hex str): md5 hash expected from the file's content
content_length (int): the expected length if provided.
Returns:
Either none if no error or a dictionary with a key error
detailing the problem.
"""
if content_length:
if content_length > self.config['max_upload_size']:
return make_error_dict(
MAX_UPLOAD_SIZE_EXCEEDED,
'Upload size limit exceeded (max %s bytes).' %
self.config['max_upload_size'],
'Please consider sending the archive in '
'multiple steps.')
length = filehandler.size
if length != content_length:
return make_error_dict(status.HTTP_412_PRECONDITION_FAILED,
'Wrong length')
if md5sum:
_md5sum = self._compute_md5(filehandler)
if _md5sum != md5sum:
return make_error_dict(
CHECKSUM_MISMATCH,
'Wrong md5 hash',
'The checksum sent %s and the actual checksum '
'%s does not match.' % (md5sum, _md5sum))
return None
def _binary_upload(self, req, headers, collection_name, deposit_id=None,
replace_metadata=False, replace_archives=False):
"""Binary upload routine.
Other than such a request, a 415 response is returned.
Args:
req (Request): the request holding information to parse
and inject in db
headers (dict): request headers formatted
collection_name (str): the associated client
deposit_id (id): deposit identifier if provided
replace_metadata (bool): 'Update or add' request to existing
deposit. If False (default), this adds new metadata request to
existing ones. Otherwise, this will replace existing metadata.
replace_archives (bool): 'Update or add' request to existing
deposit. If False (default), this adds new archive request to
existing ones. Otherwise, this will replace existing archives.
ones.
Returns:
In the optimal case a dict with the following keys:
- deposit_id (int): Deposit identifier
- deposit_date (date): Deposit date
- archive: None (no archive is provided here)
Otherwise, a dictionary with the key error and the
associated failures, either:
- 400 (bad request) if the request is not providing an external
identifier
- 403 (forbidden) if the length of the archive exceeds the
max size configured
- 412 (precondition failed) if the length or md5 hash provided
mismatch the reality of the archive
- 415 (unsupported media type) if a wrong media type is provided
"""
content_length = headers['content-length']
if not content_length:
return make_error_dict(
BAD_REQUEST,
'CONTENT_LENGTH header is mandatory',
'For archive deposit, the '
'CONTENT_LENGTH header must be sent.')
content_disposition = headers['content-disposition']
if not content_disposition:
return make_error_dict(
BAD_REQUEST,
'CONTENT_DISPOSITION header is mandatory',
'For archive deposit, the '
'CONTENT_DISPOSITION header must be sent.')
packaging = headers['packaging']
if packaging and packaging not in ACCEPT_PACKAGINGS:
return make_error_dict(
BAD_REQUEST,
'Only packaging %s is supported' %
ACCEPT_PACKAGINGS,
'The packaging provided %s is not supported' % packaging)
filehandler = req.FILES['file']
precondition_status_response = self._check_preconditions_on(
filehandler, headers['content-md5sum'], content_length)
if precondition_status_response:
return precondition_status_response
external_id = headers['slug']
# actual storage of data
archive_metadata = filehandler
deposit = self._deposit_put(deposit_id=deposit_id,
in_progress=headers['in-progress'],
external_id=external_id)
self._deposit_request_put(
deposit, {ARCHIVE_KEY: archive_metadata},
replace_metadata=replace_metadata,
replace_archives=replace_archives)
return {
'deposit_id': deposit.id,
'deposit_date': deposit.reception_date,
'archive': filehandler.name,
}
def _multipart_upload(self, req, headers, collection_name,
deposit_id=None, replace_metadata=False,
replace_archives=False):
"""Multipart upload supported with exactly:
- 1 archive (zip)
- 1 atom entry
Other than such a request, a 415 response is returned.
Args:
req (Request): the request holding information to parse
and inject in db
headers (dict): request headers formatted
collection_name (str): the associated client
deposit_id (id): deposit identifier if provided
replace_metadata (bool): 'Update or add' request to existing
deposit. If False (default), this adds new metadata request to
existing ones. Otherwise, this will replace existing metadata.
replace_archives (bool): 'Update or add' request to existing
deposit. If False (default), this adds new archive request to
existing ones. Otherwise, this will replace existing archives.
ones.
Returns:
In the optimal case a dict with the following keys:
- deposit_id (int): Deposit identifier
- deposit_date (date): Deposit date
- archive: None (no archive is provided here)
Otherwise, a dictionary with the key error and the
associated failures, either:
- 400 (bad request) if the request is not providing an external
identifier
- 412 (precondition failed) if the potentially md5 hash provided
mismatch the reality of the archive
- 415 (unsupported media type) if a wrong media type is provided
"""
external_id = headers['slug']
content_types_present = set()
data = {
'application/zip': None, # expected archive
'application/atom+xml': None,
}
for key, value in req.FILES.items():
fh = value
if fh.content_type in content_types_present:
return make_error_dict(
ERROR_CONTENT,
'Only 1 application/zip archive and 1 '
'atom+xml entry is supported (as per sword2.0 '
'specification)',
'You provided more than 1 application/zip '
'or more than 1 application/atom+xml content-disposition '
'header in the multipart deposit')
content_types_present.add(fh.content_type)
data[fh.content_type] = fh
if len(content_types_present) != 2:
return make_error_dict(
ERROR_CONTENT,
'You must provide both 1 application/zip '
'and 1 atom+xml entry for multipart deposit',
'You need to provide only 1 application/zip '
'and 1 application/atom+xml content-disposition header '
'in the multipart deposit')
filehandler = data['application/zip']
precondition_status_response = self._check_preconditions_on(
filehandler,
headers['content-md5sum'])
if precondition_status_response:
return precondition_status_response
# actual storage of data
atom_metadata = parse_xml(data['application/atom+xml'])
deposit = self._deposit_put(deposit_id=deposit_id,
in_progress=headers['in-progress'],
external_id=external_id)
deposit_request_data = {
ARCHIVE_KEY: filehandler,
METADATA_KEY: atom_metadata,
}
self._deposit_request_put(
deposit, deposit_request_data, replace_metadata, replace_archives)
return {
'deposit_id': deposit.id,
'deposit_date': deposit.reception_date,
'archive': filehandler.name,
}
def _atom_entry(self, req, headers, collection_name,
deposit_id=None,
replace_metadata=False,
replace_archives=False):
"""Atom entry deposit.
Args:
req (Request): the request holding information to parse
and inject in db
headers (dict): request headers formatted
collection_name (str): the associated client
deposit_id (id): deposit identifier if provided
replace_metadata (bool): 'Update or add' request to existing
deposit. If False (default), this adds new metadata request to
existing ones. Otherwise, this will replace existing metadata.
replace_archives (bool): 'Update or add' request to existing
deposit. If False (default), this adds new archive request to
existing ones. Otherwise, this will replace existing archives.
ones.
Returns:
In the optimal case a dict with the following keys:
- deposit_id: deposit id associated to the deposit
- deposit_date: date of the deposit
- archive: None (no archive is provided here)
Otherwise, a dictionary with the key error and the
associated failures, either:
- 400 (bad request) if the request is not providing an external
identifier
- 400 (bad request) if the request's body is empty
- 415 (unsupported media type) if a wrong media type is provided
"""
if not req.data:
return make_error_dict(
BAD_REQUEST,
'Empty body request is not supported',
'Atom entry deposit is supposed to send for metadata. '
'If the body is empty, there is no metadata.')
external_id = req.data.get(
'{http://www.w3.org/2005/Atom}external_identifier',
headers['slug'])
deposit = self._deposit_put(deposit_id=deposit_id,
in_progress=headers['in-progress'],
external_id=external_id)
self._deposit_request_put(
deposit, {METADATA_KEY: req.data},
replace_metadata, replace_archives)
return {
'deposit_id': deposit.id,
'deposit_date': deposit.reception_date,
'archive': None,
}
def _empty_post(self, req, headers, collection_name, deposit_id):
"""Empty post to finalize an empty deposit.
Args:
req (Request): the request holding information to parse
and inject in db
headers (dict): request headers formatted
collection_name (str): the associated client
deposit_id (id): deposit identifier
Returns:
Dictionary of result with the deposit's id, the date
it was completed and no archive.
"""
deposit = Deposit.objects.get(pk=deposit_id)
deposit.complete_date = timezone.now()
deposit.status = 'ready'
deposit.save()
return {
'deposit_id': deposit_id,
'deposit_date': deposit.complete_date,
'archive': None,
}
def _make_iris(self, collection_name, deposit_id):
"""Define the IRI endpoints
Args:
collection_name (str): client/collection's name
deposit_id (id): Deposit identifier
Returns:
Dictionary of keys with the iris' urls.
"""
return {
EM_IRI: reverse(
EM_IRI,
args=[collection_name, deposit_id]),
EDIT_SE_IRI: reverse(
EDIT_SE_IRI,
args=[collection_name, deposit_id]),
CONT_FILE_IRI: reverse(
CONT_FILE_IRI,
args=[collection_name, deposit_id]),
}
- def additional_checks(self, req, collection_name, deposit_id=None):
- """Permit the child class to enrich with additional checks.
+ def additional_checks(self, req, headers, collection_name,
+ deposit_id=None):
+ """Permit the child class to enrich additional checks.
Returns:
dict with 'error' detailing the problem.
"""
return {}
def checks(self, req, collection_name, deposit_id=None):
try:
self._collection = DepositCollection.objects.get(
name=collection_name)
except DepositCollection.DoesNotExist:
return make_error_dict(
NOT_FOUND,
'Unknown collection name %s' % collection_name)
username = req.user.username
if username: # unauthenticated request can have the username empty
try:
self._client = DepositClient.objects.get(username=username)
except DepositClient.DoesNotExist:
return make_error_dict(NOT_FOUND,
'Unknown client name %s' % username)
if self._collection.id not in self._client.collections:
return make_error_dict(
FORBIDDEN,
'Client %s cannot access collection %s' % (
username, collection_name))
if deposit_id:
try:
deposit = Deposit.objects.get(pk=deposit_id)
except Deposit.DoesNotExist:
return make_error_dict(
NOT_FOUND,
'Deposit with id %s does not exist' %
deposit_id)
checks = self.restrict_access(req, deposit)
if checks:
return checks
headers = self._read_headers(req)
if headers['on-behalf-of']:
return make_error_dict(MEDIATION_NOT_ALLOWED,
'Mediation is not supported.')
- checks = self.additional_checks(req, collection_name, deposit_id)
+ checks = self.additional_checks(req, headers,
+ collection_name, deposit_id)
if 'error' in checks:
return checks
return {'headers': headers}
def restrict_access(self, req, deposit=None):
if deposit:
if req.method != 'GET' and deposit.status != 'partial':
summary = "You can only act on deposit with status 'partial'"
description = "This deposit has status '%s'" % deposit.status
return make_error_dict(
BAD_REQUEST, summary=summary,
verbose_description=description)
def get(self, req, *args, **kwargs):
return make_error_response(req, METHOD_NOT_ALLOWED)
def post(self, req, *args, **kwargs):
return make_error_response(req, METHOD_NOT_ALLOWED)
def put(self, req, *args, **kwargs):
return make_error_response(req, METHOD_NOT_ALLOWED)
def delete(self, req, *args, **kwargs):
return make_error_response(req, METHOD_NOT_ALLOWED)
class SWHGetDepositAPI(SWHBaseDeposit, metaclass=ABCMeta):
"""Mixin for class to support GET method.
"""
def get(self, req, collection_name, deposit_id, format=None):
"""Endpoint to create/add resources to deposit.
Returns:
200 response when no error during routine occurred
400 if the deposit does not belong to the collection
404 if the deposit or the collection does not exist
"""
checks = self.checks(req, collection_name, deposit_id)
if 'error' in checks:
return make_error_response_from_dict(req, checks['error'])
status, content, content_type = self.process_get(
req, collection_name, deposit_id)
return HttpResponse(content, status=status, content_type=content_type)
@abstractmethod
def process_get(self, req, collection_name, deposit_id):
"""Routine to deal with the deposit's get processing.
Returns:
Tuple status, stream of content, content-type
"""
pass
class SWHPostDepositAPI(SWHBaseDeposit, metaclass=ABCMeta):
"""Mixin for class to support DELETE method.
"""
def post(self, req, collection_name, deposit_id=None, format=None):
"""Endpoint to create/add resources to deposit.
Returns:
204 response when no error during routine occurred.
400 if the deposit does not belong to the collection
404 if the deposit or the collection does not exist
"""
checks = self.checks(req, collection_name, deposit_id)
if 'error' in checks:
return make_error_response_from_dict(req, checks['error'])
headers = checks['headers']
_status, _iri_key, data = self.process_post(
req, headers, collection_name, deposit_id)
error = data.get('error')
if error:
return make_error_response_from_dict(req, error)
data['packagings'] = ACCEPT_PACKAGINGS
iris = self._make_iris(collection_name, data['deposit_id'])
data.update(iris)
response = render(req, 'deposit/deposit_receipt.xml',
context=data,
content_type='application/xml',
status=_status)
response._headers['location'] = 'Location', data[_iri_key]
return response
@abstractmethod
def process_post(self, req, headers, collection_name, deposit_id=None):
"""Routine to deal with the deposit's processing.
Returns
Tuple of:
- response status code (200, 201, etc...)
- key iri (EM_IRI, EDIT_SE_IRI, etc...)
- dictionary of the processing result
"""
pass
class SWHPutDepositAPI(SWHBaseDeposit, metaclass=ABCMeta):
"""Mixin for class to support PUT method.
"""
def put(self, req, collection_name, deposit_id, format=None):
"""Endpoint to update deposit resources.
Returns:
204 response when no error during routine occurred.
400 if the deposit does not belong to the collection
404 if the deposit or the collection does not exist
"""
checks = self.checks(req, collection_name, deposit_id)
if 'error' in checks:
return make_error_response_from_dict(req, checks['error'])
headers = checks['headers']
data = self.process_put(req, headers, collection_name, deposit_id)
error = data.get('error')
if error:
return make_error_response_from_dict(req, error)
return HttpResponse(status=status.HTTP_204_NO_CONTENT)
@abstractmethod
def process_put(self, req, headers, collection_name, deposit_id):
"""Routine to deal with updating a deposit in some way.
Returns
dictionary of the processing result
"""
pass
class SWHDeleteDepositAPI(SWHBaseDeposit, metaclass=ABCMeta):
"""Mixin for class to support DELETE method.
"""
def delete(self, req, collection_name, deposit_id):
"""Endpoint to delete some deposit's resources (archives, deposit).
Returns:
204 response when no error during routine occurred.
400 if the deposit does not belong to the collection
404 if the deposit or the collection does not exist
"""
checks = self.checks(req, collection_name, deposit_id)
if 'error' in checks:
return make_error_response_from_dict(req, checks['error'])
data = self.process_delete(req, collection_name, deposit_id)
error = data.get('error')
if error:
return make_error_response_from_dict(req, error)
return HttpResponse(status=status.HTTP_204_NO_CONTENT)
@abstractmethod
def process_delete(self, req, collection_name, deposit_id):
"""Routine to delete a resource.
This is mostly not allowed except for the
EM_IRI (cf. .api.deposit_update.SWHUpdateArchiveDeposit)
"""
pass
diff --git a/swh/deposit/api/deposit.py b/swh/deposit/api/deposit.py
index a307119d..77f73b9c 100644
--- a/swh/deposit/api/deposit.py
+++ b/swh/deposit/api/deposit.py
@@ -1,80 +1,91 @@
# Copyright (C) 2017 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from rest_framework import status
from .common import SWHPostDepositAPI
from ..config import EDIT_SE_IRI
+from ..errors import make_error_dict, BAD_REQUEST
from ..parsers import SWHFileUploadParser, SWHAtomEntryParser
from ..parsers import SWHMultiPartParser
class SWHDeposit(SWHPostDepositAPI):
"""Deposit request class defining api endpoints for sword deposit.
What's known as 'Col IRI' in the sword specification.
HTTP verbs supported: POST
"""
parser_classes = (SWHMultiPartParser,
SWHFileUploadParser,
SWHAtomEntryParser)
+ def additional_checks(self, req, headers, collection_name,
+ deposit_id=None):
+ slug = headers['slug']
+ if not slug:
+ msg = 'Missing SLUG header in request'
+ verbose_description = 'Provide in the SLUG header one identifier, for example the url pointing to the resource you are depositing.' # noqa
+ return make_error_dict(BAD_REQUEST, msg, verbose_description)
+
+ return {}
+
def process_post(self, req, headers, collection_name, deposit_id=None):
"""Create a first deposit as:
- archive deposit (1 zip)
- multipart (1 zip + 1 atom entry)
- atom entry
Args:
req (Request): the request holding the information to parse
and inject in db
collection_name (str): the associated client
Returns:
An http response (HttpResponse) according to the situation.
If everything is ok, a 201 response (created) with a
deposit receipt.
Otherwise, depending on the upload, the following errors
can be returned:
- archive deposit:
- 400 (bad request) if the request is not providing an external
identifier
- 403 (forbidden) if the length of the archive exceeds the
max size configured
- 412 (precondition failed) if the length or hash provided
mismatch the reality of the archive.
- 415 (unsupported media type) if a wrong media type is
provided
- multipart deposit:
- 400 (bad request) if the request is not providing an external
identifier
- 412 (precondition failed) if the potentially md5 hash
provided mismatch the reality of the archive
- 415 (unsupported media type) if a wrong media type is
provided
- Atom entry deposit:
- 400 (bad request) if the request is not providing an external
identifier
- 400 (bad request) if the request's body is empty
- 415 (unsupported media type) if a wrong media type is
provided
"""
assert deposit_id is None
if req.content_type == 'application/zip':
data = self._binary_upload(req, headers, collection_name)
elif req.content_type.startswith('multipart/'):
data = self._multipart_upload(req, headers, collection_name)
else:
data = self._atom_entry(req, headers, collection_name)
return status.HTTP_201_CREATED, EDIT_SE_IRI, data
diff --git a/swh/deposit/api/private/deposit_update_status.py b/swh/deposit/api/private/deposit_update_status.py
index 7fa9ada1..9f432ed0 100644
--- a/swh/deposit/api/private/deposit_update_status.py
+++ b/swh/deposit/api/private/deposit_update_status.py
@@ -1,70 +1,71 @@
# Copyright (C) 2017 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from rest_framework.parsers import JSONParser
from ..common import SWHPutDepositAPI, SWHPrivateAPIView
from ...errors import make_error_dict, BAD_REQUEST
from ...models import Deposit, DEPOSIT_STATUS_DETAIL, format_swh_id
class SWHUpdateStatusDeposit(SWHPutDepositAPI, SWHPrivateAPIView):
"""Deposit request class to update the deposit's status.
HTTP verbs supported: PUT
"""
parser_classes = (JSONParser, )
- def additional_checks(self, req, collection_name, deposit_id=None):
+ def additional_checks(self, req, headers, collection_name,
+ deposit_id=None):
"""Enrich existing checks to the default ones.
New checks:
- Ensure the status is provided
- Ensure it exists
"""
data = req.data
status = data.get('status')
if not status:
msg = 'The status key is mandatory with possible values %s' % list(
DEPOSIT_STATUS_DETAIL.keys())
return make_error_dict(BAD_REQUEST, msg)
if status not in DEPOSIT_STATUS_DETAIL:
msg = 'Possible status in %s' % list(DEPOSIT_STATUS_DETAIL.keys())
return make_error_dict(BAD_REQUEST, msg)
if status == 'success':
swh_id = data.get('revision_id')
if not swh_id:
msg = 'Updating status to %s requires a revision_id key' % (
status, )
return make_error_dict(BAD_REQUEST, msg)
return {}
def restrict_access(self, req, deposit=None):
"""Remove restriction modification to 'partial' deposit.
Update is possible regardless of the existing status.
"""
return None
def process_put(self, req, headers, collection_name, deposit_id):
"""Update the deposit's status
Returns:
204 No content
"""
deposit = Deposit.objects.get(pk=deposit_id)
deposit.status = req.data['status'] # checks already done before
swh_id = req.data.get('revision_id')
if swh_id:
deposit.swh_id = format_swh_id(collection_name, swh_id)
deposit.save()
return {}
diff --git a/swh/deposit/tests/api/test_deposit_atom.py b/swh/deposit/tests/api/test_deposit_atom.py
index c82e38b6..2525d37c 100644
--- a/swh/deposit/tests/api/test_deposit_atom.py
+++ b/swh/deposit/tests/api/test_deposit_atom.py
@@ -1,258 +1,273 @@
# Copyright (C) 2017 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from django.core.urlresolvers import reverse
from io import BytesIO
from rest_framework import status
from rest_framework.test import APITestCase
from swh.deposit.config import COL_IRI
from swh.deposit.models import Deposit, DepositRequest
from swh.deposit.parsers import parse_xml
from ..common import BasicTestCase, WithAuthTestCase
class DepositAtomEntryTestCase(APITestCase, WithAuthTestCase, BasicTestCase):
"""Try and post atom entry deposit.
"""
def setUp(self):
super().setUp()
self.atom_entry_data0 = b"""
Awesome Compiler
hal
urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a
%s
2017-10-07T15:17:08Z
some awesome author
something
awesome-compiler
This is an awesome compiler destined to
awesomely compile stuff
and other stuff
compiler,programming,language
2005-10-07T17:17:08Z
2005-10-07T17:17:08Z
release note
related link
Awesome
https://hoster.org/awesome-compiler
GNU/Linux
0.0.1
running
all
"""
self.atom_entry_data1 = b"""
hal
urn:uuid:2225c695-cfb8-4ebb-aaaa-80da344efa6a
2017-10-07T15:17:08Z
some awesome author
something
awesome-compiler
This is an awesome compiler destined to
awesomely compile stuff
and other stuff
compiler,programming,language
2005-10-07T17:17:08Z
2005-10-07T17:17:08Z
release note
related link
Awesome
https://hoster.org/awesome-compiler
GNU/Linux
0.0.1
running
all
"""
self.atom_entry_data2 = b"""
%s
"""
self.atom_entry_data_empty_body = b"""
"""
self.atom_entry_data3 = b"""
something
"""
self.atom_entry_tei = b"""HAL TEI export of hal-01587083CCSDDistributed under a Creative Commons Attribution 4.0 International LicenseHAL API platform
questionnaire software metadataMoraneGruenpeter7de56c632362954fa84172cad80afe4einria.fr1556733MoraneGruenpeterf85a43a5fb4a2e0778a77e017f28c8fdgmail.com2017-09-29 11:21:322017-10-03 17:20:132017-10-03 17:20:132017-09-292017-09-29contributorMoraneGruenpeterf85a43a5fb4a2e0778a77e017f28c8fdgmail.comCCSDhal-01587083https://hal.inria.fr/hal-01587083gruenpeter:hal-0158708320172017questionnaire software metadataMoraneGruenpeter7de56c632362954fa84172cad80afe4einria.fr1556733EnglishComputer Science [cs]SoftwareIRILLInitiative pour la Recherche et l'Innovation sur le Logiciel Libre[https://www.irill.org/]Universite Pierre et Marie Curie - Paris 6UPMC4 place Jussieu - 75005 Paris[http://www.upmc.fr/]Institut National de Recherche en Informatique et en AutomatiqueInriaDomaine de VoluceauRocquencourt - BP 10578153 Le Chesnay Cedex[http://www.inria.fr/en/]Universite Paris Diderot - Paris 7UPD75 rue Thomas-Mann - 75205 Paris cedex 13[http://www.univ-paris-diderot.fr]""" # noqa
self.atom_entry_data_badly_formatted = b"""
"""
def test_post_deposit_atom_empty_body_request(self):
response = self.client.post(
reverse(COL_IRI, args=[self.collection.name]),
content_type='application/atom+xml;type=entry',
data=self.atom_entry_data_empty_body)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_post_deposit_atom_badly_formatted_is_a_bad_request(self):
response = self.client.post(
reverse(COL_IRI, args=[self.collection.name]),
content_type='application/atom+xml;type=entry',
data=self.atom_entry_data_badly_formatted)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+ def test_post_deposit_atom_without_slug_header_is_bad_request(self):
+ url = reverse(COL_IRI, args=[self.collection.name])
+
+ # when
+ response = self.client.post(
+ url,
+ content_type='application/atom+xml;type=entry',
+ data=self.atom_entry_data0,
+ # + headers
+ HTTP_IN_PROGRESS='false')
+
+ self.assertIn(b'Missing SLUG header', response.content)
+ self.assertEqual(response.status_code,
+ status.HTTP_400_BAD_REQUEST)
+
def test_post_deposit_atom_unknown_collection(self):
response = self.client.post(
reverse(COL_IRI, args=['unknown-one']),
content_type='application/atom+xml;type=entry',
data=self.atom_entry_data3,
HTTP_SLUG='something')
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
def test_post_deposit_atom_entry_initial(self):
"""One deposit upload as atom entry
"""
# given
external_id = 'urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a'
with self.assertRaises(Deposit.DoesNotExist):
Deposit.objects.get(external_id=external_id)
atom_entry_data = self.atom_entry_data0 % external_id.encode('utf-8')
# when
response = self.client.post(
reverse(COL_IRI, args=[self.collection.name]),
content_type='application/atom+xml;type=entry',
data=atom_entry_data,
+ HTTP_SLUG='external-id',
HTTP_IN_PROGRESS='false')
# then
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
response_content = parse_xml(BytesIO(response.content))
deposit_id = response_content[
'{http://www.w3.org/2005/Atom}deposit_id']
deposit = Deposit.objects.get(pk=deposit_id)
self.assertEqual(deposit.collection, self.collection)
self.assertEqual(deposit.external_id, external_id)
self.assertEqual(deposit.status, 'ready')
self.assertEqual(deposit.client, self.user)
# one associated request to a deposit
deposit_request = DepositRequest.objects.get(deposit=deposit)
self.assertIsNotNone(deposit_request.metadata)
self.assertFalse(bool(deposit_request.archive))
def test_post_deposit_atom_entry_tei(self):
"""One deposit upload as atom entry
"""
# given
external_id = 'urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a'
with self.assertRaises(Deposit.DoesNotExist):
Deposit.objects.get(external_id=external_id)
atom_entry_data = self.atom_entry_tei
# when
response = self.client.post(
reverse(COL_IRI, args=[self.collection.name]),
content_type='application/atom+xml;type=entry',
data=atom_entry_data,
HTTP_SLUG=external_id,
HTTP_IN_PROGRESS='false')
# then
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
response_content = parse_xml(BytesIO(response.content))
deposit_id = response_content[
'{http://www.w3.org/2005/Atom}deposit_id']
deposit = Deposit.objects.get(pk=deposit_id)
self.assertEqual(deposit.collection, self.collection)
self.assertEqual(deposit.external_id, external_id)
self.assertEqual(deposit.status, 'ready')
self.assertEqual(deposit.client, self.user)
# one associated request to a deposit
deposit_request = DepositRequest.objects.get(deposit=deposit)
self.assertIsNotNone(deposit_request.metadata)
self.assertFalse(bool(deposit_request.archive))
def test_post_deposit_atom_entry_multiple_steps(self):
"""Test one deposit upload."""
# given
external_id = 'urn:uuid:2225c695-cfb8-4ebb-aaaa-80da344efa6a'
with self.assertRaises(Deposit.DoesNotExist):
deposit = Deposit.objects.get(external_id=external_id)
# when
response = self.client.post(
reverse(COL_IRI, args=[self.collection.name]),
content_type='application/atom+xml;type=entry',
data=self.atom_entry_data1,
HTTP_IN_PROGRESS='True',
HTTP_SLUG=external_id)
# then
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
response_content = parse_xml(BytesIO(response.content))
deposit_id = response_content[
'{http://www.w3.org/2005/Atom}deposit_id']
deposit = Deposit.objects.get(pk=deposit_id)
self.assertEqual(deposit.collection, self.collection)
self.assertEqual(deposit.external_id, external_id)
self.assertEqual(deposit.status, 'partial')
self.assertEqual(deposit.client, self.user)
# one associated request to a deposit
deposit_requests = DepositRequest.objects.filter(deposit=deposit)
self.assertEqual(len(deposit_requests), 1)
atom_entry_data = self.atom_entry_data2 % external_id.encode('utf-8')
update_uri = response._headers['location'][1]
# when updating the first deposit post
response = self.client.post(
update_uri,
content_type='application/atom+xml;type=entry',
data=atom_entry_data,
- HTTP_IN_PROGRESS='False',
- HTTP_SLUG=external_id)
+ HTTP_IN_PROGRESS='False')
# then
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
response_content = parse_xml(BytesIO(response.content))
deposit_id = response_content[
'{http://www.w3.org/2005/Atom}deposit_id']
deposit = Deposit.objects.get(pk=deposit_id)
self.assertEqual(deposit.collection, self.collection)
self.assertEqual(deposit.external_id, external_id)
self.assertEqual(deposit.status, 'ready')
self.assertEqual(deposit.client, self.user)
self.assertEqual(len(Deposit.objects.all()), 1)
# now 2 associated requests to a same deposit
deposit_requests = DepositRequest.objects.filter(deposit=deposit)
self.assertEqual(len(deposit_requests), 2)
for deposit_request in deposit_requests:
actual_metadata = deposit_request.metadata
self.assertIsNotNone(actual_metadata)
self.assertFalse(bool(deposit_request.archive))
diff --git a/swh/deposit/tests/api/test_deposit_binary.py b/swh/deposit/tests/api/test_deposit_binary.py
index d948b2b0..1894c75b 100644
--- a/swh/deposit/tests/api/test_deposit_binary.py
+++ b/swh/deposit/tests/api/test_deposit_binary.py
@@ -1,663 +1,684 @@
# Copyright (C) 2017 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import hashlib
from django.core.files.uploadedfile import InMemoryUploadedFile
from django.core.urlresolvers import reverse
from io import BytesIO
from rest_framework import status
from rest_framework.test import APITestCase
from swh.deposit.config import COL_IRI, EM_IRI
from swh.deposit.models import Deposit, DepositRequest
from swh.deposit.parsers import parse_xml
from ..common import BasicTestCase, WithAuthTestCase
class DepositNoAuthCase(APITestCase, BasicTestCase):
"""Deposit access are protected with basic authentication.
"""
def test_post_will_fail_with_401(self):
"""Without authentication, endpoint refuses access with 401 response
"""
url = reverse(COL_IRI, args=[self.collection.name])
data_text = b'some content'
md5sum = hashlib.md5(data_text).hexdigest()
external_id = 'some-external-id-1'
# when
response = self.client.post(
url,
content_type='application/zip', # as zip
data=data_text,
# + headers
HTTP_SLUG=external_id,
HTTP_CONTENT_MD5=md5sum,
HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
HTTP_IN_PROGRESS='false',
HTTP_CONTENT_LENGTH=len(data_text),
HTTP_CONTENT_DISPOSITION='attachment; filename=filename0')
# then
self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
class DepositTestCase(APITestCase, WithAuthTestCase, BasicTestCase):
"""Try and upload one single deposit
"""
def setUp(self):
super().setUp()
self.atom_entry_data0 = b"""
Awesome Compiler
hal
urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a
%s
2017-10-07T15:17:08Z
some awesome author
something
awesome-compiler
This is an awesome compiler destined to
awesomely compile stuff
and other stuff
compiler,programming,language
2005-10-07T17:17:08Z
2005-10-07T17:17:08Z
release note
related link
Awesome
https://hoster.org/awesome-compiler
GNU/Linux
0.0.1
running
all
"""
self.atom_entry_data1 = b"""
hal
urn:uuid:2225c695-cfb8-4ebb-aaaa-80da344efa6a
2017-10-07T15:17:08Z
some awesome author
something
awesome-compiler
This is an awesome compiler destined to
awesomely compile stuff
and other stuff
compiler,programming,language
2005-10-07T17:17:08Z
2005-10-07T17:17:08Z
release note
related link
Awesome
https://hoster.org/awesome-compiler
GNU/Linux
0.0.1
running
all
"""
self.atom_entry_data2 = b"""
%s
"""
self.atom_entry_data_empty_body = b"""
"""
self.atom_entry_data3 = b"""
something
"""
self.data_atom_entry_ok = b"""
Title
urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a
2005-10-07T17:17:08Z
Contributor
The abstract
The abstract
Access Rights
Alternative Title
Date Available
Bibliographic Citation # noqa
Contributor
Description
Has Part
Has Version
Identifier
Is Part Of
Publisher
References
Rights Holder
Source
Title
Type
"""
+ def test_post_deposit_binary_without_slug_header_is_bad_request(self):
+ url = reverse(COL_IRI, args=[self.collection.name])
+ data_text = b'some content'
+ md5sum = hashlib.md5(data_text).hexdigest()
+
+ # when
+ response = self.client.post(
+ url,
+ content_type='application/zip', # as zip
+ data=data_text,
+ # + headers
+ HTTP_CONTENT_MD5=md5sum,
+ HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
+ HTTP_IN_PROGRESS='false',
+ HTTP_CONTENT_LENGTH=len(data_text),
+ HTTP_CONTENT_DISPOSITION='attachment; filename=filename0')
+
+ self.assertIn(b'Missing SLUG header', response.content)
+ self.assertEqual(response.status_code,
+ status.HTTP_400_BAD_REQUEST)
+
def test_post_deposit_binary_upload_final_and_status_check(self):
"""Binary upload should be accepted
"""
# given
url = reverse(COL_IRI, args=[self.collection.name])
data_text = b'some content'
md5sum = hashlib.md5(data_text).hexdigest()
external_id = 'some-external-id-1'
# when
response = self.client.post(
url,
content_type='application/zip', # as zip
data=data_text,
# + headers
HTTP_SLUG=external_id,
HTTP_CONTENT_MD5=md5sum,
HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
HTTP_IN_PROGRESS='false',
HTTP_CONTENT_LENGTH=len(data_text),
HTTP_CONTENT_DISPOSITION='attachment; filename=filename0')
# then
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
response_content = parse_xml(BytesIO(response.content))
deposit_id = response_content[
'{http://www.w3.org/2005/Atom}deposit_id']
deposit = Deposit.objects.get(pk=deposit_id)
self.assertEqual(deposit.status, 'ready')
self.assertEqual(deposit.external_id, external_id)
self.assertEqual(deposit.collection, self.collection)
self.assertEqual(deposit.client, self.user)
self.assertIsNone(deposit.swh_id)
deposit_request = DepositRequest.objects.get(deposit=deposit)
self.assertEquals(deposit_request.deposit, deposit)
self.assertRegex(deposit_request.archive.name, 'filename0')
response_content = parse_xml(BytesIO(response.content))
self.assertEqual(
response_content['{http://www.w3.org/2005/Atom}deposit_archive'],
'filename0')
self.assertEqual(
response_content['{http://www.w3.org/2005/Atom}deposit_id'],
deposit.id)
edit_se_iri = reverse('edit_se_iri',
args=[self.collection.name, deposit.id])
self.assertEqual(response._headers['location'],
('Location', edit_se_iri))
def test_post_deposit_binary_upload_only_supports_zip(self):
"""Binary upload only supports application/zip (for now)...
"""
# given
url = reverse(COL_IRI, args=[self.collection.name])
data_text = b'some content'
md5sum = hashlib.md5(data_text).hexdigest()
external_id = 'some-external-id-1'
# when
response = self.client.post(
url,
content_type='application/octet-stream',
data=data_text,
# + headers
HTTP_SLUG=external_id,
HTTP_CONTENT_MD5=md5sum,
HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
HTTP_IN_PROGRESS='false',
HTTP_CONTENT_LENGTH=len(data_text),
HTTP_CONTENT_DISPOSITION='attachment; filename=filename0')
# then
self.assertEqual(response.status_code,
status.HTTP_415_UNSUPPORTED_MEDIA_TYPE)
with self.assertRaises(Deposit.DoesNotExist):
Deposit.objects.get(external_id=external_id)
def test_post_deposit_binary_fails_if_unsupported_packaging_header(
self):
"""Binary upload must have content_disposition header provided...
"""
# given
url = reverse(COL_IRI, args=[self.collection.name])
data_text = b'some content'
md5sum = hashlib.md5(data_text).hexdigest()
external_id = 'some-external-id'
# when
response = self.client.post(
url,
content_type='application/zip',
data=data_text,
# + headers
HTTP_SLUG=external_id,
HTTP_CONTENT_MD5=md5sum,
HTTP_PACKAGING='something-unsupported',
HTTP_CONTENT_LENGTH=len(data_text),
HTTP_CONTENT_DISPOSITION='attachment; filename=filename0')
# then
self.assertEqual(response.status_code,
status.HTTP_400_BAD_REQUEST)
with self.assertRaises(Deposit.DoesNotExist):
Deposit.objects.get(external_id=external_id)
def test_post_deposit_binary_upload_fail_if_no_content_disposition_header(
self):
"""Binary upload must have content_disposition header provided...
"""
# given
url = reverse(COL_IRI, args=[self.collection.name])
data_text = b'some content'
md5sum = hashlib.md5(data_text).hexdigest()
external_id = 'some-external-id'
# when
response = self.client.post(
url,
content_type='application/zip',
data=data_text,
# + headers
HTTP_SLUG=external_id,
HTTP_CONTENT_MD5=md5sum,
HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
HTTP_IN_PROGRESS='false',
HTTP_CONTENT_LENGTH=len(data_text))
# then
self.assertEqual(response.status_code,
status.HTTP_400_BAD_REQUEST)
with self.assertRaises(Deposit.DoesNotExist):
Deposit.objects.get(external_id=external_id)
def test_post_deposit_mediation_not_supported(self):
"""Binary upload only supports application/zip (for now)...
"""
# given
url = reverse(COL_IRI, args=[self.collection.name])
data_text = b'some content'
md5sum = hashlib.md5(data_text).hexdigest()
external_id = 'some-external-id-1'
# when
response = self.client.post(
url,
content_type='application/zip',
data=data_text,
# + headers
HTTP_SLUG=external_id,
HTTP_CONTENT_MD5=md5sum,
HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
HTTP_IN_PROGRESS='false',
HTTP_CONTENT_LENGTH=len(data_text),
HTTP_ON_BEHALF_OF='someone',
HTTP_CONTENT_DISPOSITION='attachment; filename=filename0')
# then
self.assertEqual(response.status_code,
status.HTTP_412_PRECONDITION_FAILED)
with self.assertRaises(Deposit.DoesNotExist):
Deposit.objects.get(external_id=external_id)
# FIXME: Test this scenario (need a way to override the default
# size limit in test scenario)
# def test_post_deposit_binary_upload_fail_if_upload_size_limit_exceeded(
# self):
# """Binary upload must not exceed the limit set up...
# """
# # given
# url = reverse(COL_IRI, args=[self.collection.name])
# data_text = b'some content'
# md5sum = hashlib.md5(data_text).hexdigest()
# external_id = 'some-external-id'
# # when
# response = self.client.post(
# url,
# content_type='application/zip',
# data=data_text,
# # + headers
# HTTP_SLUG=external_id,
# HTTP_CONTENT_MD5=md5sum,
# HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
# HTTP_IN_PROGRESS='false',
# CONTENT_LENGTH=len(data_text),
# HTTP_CONTENT_DISPOSITION='attachment; filename=filename0')
# # then
# self.assertEqual(response.status_code,
# status.HTTP_403_FORBIDDEN)
# with self.assertRaises(Deposit.DoesNotExist):
# Deposit.objects.get(external_id=external_id)
def test_post_deposit_2_post_2_different_deposits(self):
"""Making 2 post requests result in 2 different deposit
"""
url = reverse(COL_IRI, args=[self.collection.name])
data_text = b'some content'
md5sum = hashlib.md5(data_text).hexdigest()
# when
response = self.client.post(
url,
content_type='application/zip', # as zip
data=data_text,
# + headers
HTTP_SLUG='some-external-id-1',
HTTP_CONTENT_MD5=md5sum,
HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
HTTP_IN_PROGRESS='false',
HTTP_CONTENT_LENGTH=len(data_text),
HTTP_CONTENT_DISPOSITION='attachment; filename=filename0')
# then
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
response_content = parse_xml(BytesIO(response.content))
deposit_id = response_content[
'{http://www.w3.org/2005/Atom}deposit_id']
deposit = Deposit.objects.get(pk=deposit_id)
deposits = Deposit.objects.all()
self.assertEqual(len(deposits), 1)
self.assertEqual(deposits[0], deposit)
# second post
response = self.client.post(
url,
content_type='application/zip', # as zip
data=data_text,
# + headers
HTTP_SLUG='another-external-id',
HTTP_CONTENT_MD5=md5sum,
HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
HTTP_IN_PROGRESS='false',
HTTP_CONTENT_LENGTH=len(data_text),
HTTP_CONTENT_DISPOSITION='attachment; filename=filename1')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
response_content = parse_xml(BytesIO(response.content))
deposit_id2 = response_content[
'{http://www.w3.org/2005/Atom}deposit_id']
deposit2 = Deposit.objects.get(pk=deposit_id2)
self.assertNotEqual(deposit, deposit2)
deposits = Deposit.objects.all().order_by('id')
self.assertEqual(len(deposits), 2)
self.assertEqual(list(deposits), [deposit, deposit2])
def test_post_deposit_binary_and_post_to_add_another_archive(self):
"""One post to post a binary deposit, One other post to update the
first deposit.
"""
# given
url = reverse(COL_IRI, args=[self.collection.name])
external_id = 'some-external-id-1'
# 1st archive to upload
data_text0 = b'some other content'
md5sum0 = hashlib.md5(data_text0).hexdigest()
# when
response = self.client.post(
url,
content_type='application/zip', # as zip
data=data_text0,
# + headers
HTTP_SLUG=external_id,
HTTP_CONTENT_MD5=md5sum0,
HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
HTTP_IN_PROGRESS='true',
HTTP_CONTENT_LENGTH=len(data_text0),
HTTP_CONTENT_DISPOSITION='attachment; filename=filename0')
# then
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
response_content = parse_xml(BytesIO(response.content))
deposit_id = response_content[
'{http://www.w3.org/2005/Atom}deposit_id']
deposit = Deposit.objects.get(pk=deposit_id)
self.assertEqual(deposit.status, 'partial')
self.assertEqual(deposit.external_id, external_id)
self.assertEqual(deposit.collection, self.collection)
self.assertEqual(deposit.client, self.user)
self.assertIsNone(deposit.swh_id)
deposit_request = DepositRequest.objects.get(deposit=deposit)
self.assertEquals(deposit_request.deposit, deposit)
self.assertEquals(deposit_request.type.name, 'archive')
self.assertRegex(deposit_request.archive.name, 'filename0')
# 2nd archive to upload
data_text = b'second archive uploaded'
md5sum1 = hashlib.md5(data_text).hexdigest()
# uri to update the content
update_uri = reverse(EM_IRI, args=[self.collection.name, deposit_id])
# adding another archive for the deposit
response = self.client.post(
update_uri,
content_type='application/zip', # as zip
data=data_text,
# + headers
HTTP_SLUG=external_id,
HTTP_CONTENT_MD5=md5sum1,
HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
HTTP_IN_PROGRESS='false',
HTTP_CONTENT_LENGTH=len(data_text),
HTTP_CONTENT_DISPOSITION='attachment; filename=filename1')
deposit = Deposit.objects.get(pk=deposit_id)
self.assertEqual(deposit.status, 'ready')
self.assertEqual(deposit.external_id, external_id)
self.assertEqual(deposit.collection, self.collection)
self.assertEqual(deposit.client, self.user)
self.assertIsNone(deposit.swh_id)
deposit_requests = list(DepositRequest.objects.filter(deposit=deposit).
order_by('id'))
# 2 deposit requests for the same deposit
self.assertEquals(len(deposit_requests), 2)
self.assertEquals(deposit_requests[0].deposit, deposit)
self.assertEquals(deposit_requests[0].type.name, 'archive')
self.assertRegex(deposit_requests[0].archive.name, 'filename0')
self.assertEquals(deposit_requests[1].deposit, deposit)
self.assertEquals(deposit_requests[1].type.name, 'archive')
self.assertRegex(deposit_requests[1].archive.name, 'filename1')
# only 1 deposit in db
deposits = Deposit.objects.all()
self.assertEqual(len(deposits), 1)
def test_post_deposit_then_post_or_put_is_refused_when_status_ready(self):
"""When a deposit is complete, updating/adding new data to it is
forbidden.
"""
url = reverse(COL_IRI, args=[self.collection.name])
external_id = 'some-external-id-1'
# 1st archive to upload
data_text0 = b'some other content'
md5sum0 = hashlib.md5(data_text0).hexdigest()
# when
response = self.client.post(
url,
content_type='application/zip', # as zip
data=data_text0,
# + headers
HTTP_SLUG=external_id,
HTTP_CONTENT_MD5=md5sum0,
HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
HTTP_IN_PROGRESS='false',
HTTP_CONTENT_LENGTH=len(data_text0),
HTTP_CONTENT_DISPOSITION='attachment; filename=filename0')
# then
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
response_content = parse_xml(BytesIO(response.content))
deposit_id = response_content[
'{http://www.w3.org/2005/Atom}deposit_id']
deposit = Deposit.objects.get(pk=deposit_id)
self.assertEqual(deposit.status, 'ready')
self.assertEqual(deposit.external_id, external_id)
self.assertEqual(deposit.collection, self.collection)
self.assertEqual(deposit.client, self.user)
self.assertIsNone(deposit.swh_id)
deposit_request = DepositRequest.objects.get(deposit=deposit)
self.assertEquals(deposit_request.deposit, deposit)
self.assertRegex(deposit_request.archive.name, 'filename0')
# updating/adding is forbidden
# uri to update the content
edit_se_iri = reverse(
'edit_se_iri', args=[self.collection.name, deposit_id])
em_iri = reverse(
'em_iri', args=[self.collection.name, deposit_id])
# Testing all update/add endpoint should fail
# since the status is ready
# replacing file is no longer possible since the deposit's
# status is ready
r = self.client.put(
em_iri,
content_type='application/zip',
data=data_text0,
HTTP_SLUG=external_id,
HTTP_CONTENT_MD5=md5sum0,
HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
HTTP_IN_PROGRESS='false',
HTTP_CONTENT_LENGTH=len(data_text0),
HTTP_CONTENT_DISPOSITION='attachment; filename=filename0')
self.assertEquals(r.status_code, status.HTTP_400_BAD_REQUEST)
# adding file is no longer possible since the deposit's status
# is ready
r = self.client.post(
em_iri,
content_type='application/zip',
data=data_text0,
HTTP_SLUG=external_id,
HTTP_CONTENT_MD5=md5sum0,
HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
HTTP_IN_PROGRESS='false',
HTTP_CONTENT_LENGTH=len(data_text0),
HTTP_CONTENT_DISPOSITION='attachment; filename=filename0')
self.assertEquals(r.status_code, status.HTTP_400_BAD_REQUEST)
# replacing metadata is no longer possible since the deposit's
# status is ready
r = self.client.put(
edit_se_iri,
content_type='application/atom+xml;type=entry',
data=self.data_atom_entry_ok,
HTTP_SLUG=external_id,
HTTP_CONTENT_LENGTH=len(self.data_atom_entry_ok))
self.assertEquals(r.status_code, status.HTTP_400_BAD_REQUEST)
# adding new metadata is no longer possible since the
# deposit's status is ready
r = self.client.post(
edit_se_iri,
content_type='application/atom+xml;type=entry',
data=self.data_atom_entry_ok,
HTTP_SLUG=external_id,
HTTP_CONTENT_LENGTH=len(self.data_atom_entry_ok))
self.assertEquals(r.status_code, status.HTTP_400_BAD_REQUEST)
archive_content = b'some content representing archive'
archive = InMemoryUploadedFile(
BytesIO(archive_content),
field_name='archive0',
name='archive0',
content_type='application/zip',
size=len(archive_content),
charset=None)
atom_entry = InMemoryUploadedFile(
BytesIO(self.data_atom_entry_ok),
field_name='atom0',
name='atom0',
content_type='application/atom+xml; charset="utf-8"',
size=len(self.data_atom_entry_ok),
charset='utf-8')
# replacing multipart metadata is no longer possible since the
# deposit's status is ready
r = self.client.put(
edit_se_iri,
format='multipart',
data={
'archive': archive,
'atom_entry': atom_entry,
})
self.assertEquals(r.status_code, status.HTTP_400_BAD_REQUEST)
# adding new metadata is no longer possible since the
# deposit's status is ready
r = self.client.post(
edit_se_iri,
format='multipart',
data={
'archive': archive,
'atom_entry': atom_entry,
})
self.assertEquals(r.status_code, status.HTTP_400_BAD_REQUEST)
diff --git a/swh/deposit/tests/api/test_deposit_multipart.py b/swh/deposit/tests/api/test_deposit_multipart.py
index f002f201..f00f2afb 100644
--- a/swh/deposit/tests/api/test_deposit_multipart.py
+++ b/swh/deposit/tests/api/test_deposit_multipart.py
@@ -1,281 +1,318 @@
# Copyright (C) 2017 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from django.core.files.uploadedfile import InMemoryUploadedFile
from django.core.urlresolvers import reverse
from io import BytesIO
from rest_framework import status
from rest_framework.test import APITestCase
from swh.deposit.config import COL_IRI
from swh.deposit.models import Deposit, DepositRequest
from swh.deposit.parsers import parse_xml
from ..common import BasicTestCase, WithAuthTestCase
class DepositMultipartTestCase(APITestCase, WithAuthTestCase, BasicTestCase):
"""Post multipart deposit scenario
"""
def setUp(self):
super().setUp()
self.data_atom_entry_ok = b"""
Title
urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a
2005-10-07T17:17:08Z
Contributor
The abstract
The abstract
Access Rights
Alternative Title
Date Available
Bibliographic Citation # noqa
Contributor
Description
Has Part
Has Version
Identifier
Is Part Of
Publisher
References
Rights Holder
Source
Title
Type
"""
self.data_atom_entry_update_in_place = """
urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa7b
Title
Type
"""
+ def test_post_deposit_multipart_without_slug_header_is_bad_request(self):
+ # given
+ url = reverse(COL_IRI, args=[self.collection.name])
+ data_atom_entry = self.data_atom_entry_ok
+
+ archive_content = b'some content representing archive'
+ archive = InMemoryUploadedFile(
+ BytesIO(archive_content),
+ field_name='archive0',
+ name='archive0',
+ content_type='application/zip',
+ size=len(archive_content),
+ charset=None)
+
+ atom_entry = InMemoryUploadedFile(
+ BytesIO(data_atom_entry),
+ field_name='atom0',
+ name='atom0',
+ content_type='application/atom+xml; charset="utf-8"',
+ size=len(data_atom_entry),
+ charset='utf-8')
+
+ # when
+ response = self.client.post(
+ url,
+ format='multipart',
+ data={
+ 'archive': archive,
+ 'atom_entry': atom_entry,
+ },
+ # + headers
+ HTTP_IN_PROGRESS='false')
+
+ self.assertIn(b'Missing SLUG header', response.content)
+ self.assertEqual(response.status_code,
+ status.HTTP_400_BAD_REQUEST)
+
def test_post_deposit_multipart(self):
"""one multipart deposit should be accepted
"""
# given
url = reverse(COL_IRI, args=[self.collection.name])
# from django.core.files import uploadedfile
data_atom_entry = self.data_atom_entry_ok
archive_content = b'some content representing archive'
archive = InMemoryUploadedFile(
BytesIO(archive_content),
field_name='archive0',
name='archive0',
content_type='application/zip',
size=len(archive_content),
charset=None)
atom_entry = InMemoryUploadedFile(
BytesIO(data_atom_entry),
field_name='atom0',
name='atom0',
content_type='application/atom+xml; charset="utf-8"',
size=len(data_atom_entry),
charset='utf-8')
external_id = 'external-id'
# when
response = self.client.post(
url,
format='multipart',
data={
'archive': archive,
'atom_entry': atom_entry,
},
# + headers
HTTP_IN_PROGRESS='false',
HTTP_SLUG=external_id)
# then
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
response_content = parse_xml(BytesIO(response.content))
deposit_id = response_content[
'{http://www.w3.org/2005/Atom}deposit_id']
deposit = Deposit.objects.get(pk=deposit_id)
self.assertEqual(deposit.status, 'ready')
self.assertEqual(deposit.external_id, external_id)
self.assertEqual(deposit.collection, self.collection)
self.assertEqual(deposit.client, self.user)
self.assertIsNone(deposit.swh_id)
deposit_requests = DepositRequest.objects.filter(deposit=deposit)
self.assertEquals(len(deposit_requests), 2)
for deposit_request in deposit_requests:
self.assertEquals(deposit_request.deposit, deposit)
if deposit_request.type.name == 'archive':
self.assertRegex(deposit_request.archive.name, 'archive0')
else:
self.assertEquals(
deposit_request.metadata[
'{http://www.w3.org/2005/Atom}id'],
'urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a')
def test_post_deposit_multipart_put_to_replace_metadata(self):
"""One multipart deposit followed by a metadata update should be
accepted
"""
# given
url = reverse(COL_IRI, args=[self.collection.name])
data_atom_entry = self.data_atom_entry_ok
archive_content = b'some content representing archive'
archive = InMemoryUploadedFile(
BytesIO(archive_content),
field_name='archive0',
name='archive0',
content_type='application/zip',
size=len(archive_content),
charset=None)
atom_entry = InMemoryUploadedFile(
BytesIO(data_atom_entry),
field_name='atom0',
name='atom0',
content_type='application/atom+xml; charset="utf-8"',
size=len(data_atom_entry),
charset='utf-8')
external_id = 'external-id'
# when
response = self.client.post(
url,
format='multipart',
data={
'archive': archive,
'atom_entry': atom_entry,
},
# + headers
HTTP_IN_PROGRESS='true',
HTTP_SLUG=external_id)
# then
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
response_content = parse_xml(BytesIO(response.content))
deposit_id = response_content[
'{http://www.w3.org/2005/Atom}deposit_id']
deposit = Deposit.objects.get(pk=deposit_id)
self.assertEqual(deposit.status, 'partial')
self.assertEqual(deposit.external_id, external_id)
self.assertEqual(deposit.collection, self.collection)
self.assertEqual(deposit.client, self.user)
self.assertIsNone(deposit.swh_id)
deposit_requests = DepositRequest.objects.filter(deposit=deposit)
self.assertEquals(len(deposit_requests), 2)
for deposit_request in deposit_requests:
self.assertEquals(deposit_request.deposit, deposit)
if deposit_request.type.name == 'archive':
self.assertRegex(deposit_request.archive.name, 'archive0')
else:
self.assertEquals(
deposit_request.metadata[
'{http://www.w3.org/2005/Atom}id'],
'urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a')
replace_metadata_uri = response._headers['location'][1]
response = self.client.put(
replace_metadata_uri,
content_type='application/atom+xml;type=entry',
data=self.data_atom_entry_update_in_place,
HTTP_IN_PROGRESS='false')
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
# deposit_id did not change
deposit = Deposit.objects.get(pk=deposit_id)
self.assertEqual(deposit.status, 'ready')
self.assertEqual(deposit.external_id, external_id)
self.assertEqual(deposit.collection, self.collection)
self.assertEqual(deposit.client, self.user)
self.assertIsNone(deposit.swh_id)
deposit_requests = DepositRequest.objects.filter(deposit=deposit)
self.assertEquals(len(deposit_requests), 2)
for deposit_request in deposit_requests:
self.assertEquals(deposit_request.deposit, deposit)
if deposit_request.type.name == 'archive':
self.assertRegex(deposit_request.archive.name, 'archive0')
else:
self.assertEquals(
deposit_request.metadata[
'{http://www.w3.org/2005/Atom}id'],
'urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa7b')
# FAILURE scenarios
def test_post_deposit_multipart_only_archive_and_atom_entry(self):
"""Multipart deposit only accepts one archive and one atom+xml"""
# given
url = reverse(COL_IRI, args=[self.collection.name])
# from django.core.files import uploadedfile
archive_content = b'some content representing archive'
archive = InMemoryUploadedFile(BytesIO(archive_content),
field_name='archive0',
name='archive0',
content_type='application/zip',
size=len(archive_content),
charset=None)
other_archive_content = b"some-other-content"
other_archive = InMemoryUploadedFile(BytesIO(other_archive_content),
field_name='atom0',
name='atom0',
content_type='application/zip',
size=len(other_archive_content),
charset='utf-8')
# when
response = self.client.post(
url,
format='multipart',
data={
'archive': archive,
'atom_entry': other_archive,
},
# + headers
HTTP_IN_PROGRESS='false',
HTTP_SLUG='external-id')
# then
self.assertEqual(response.status_code,
status.HTTP_415_UNSUPPORTED_MEDIA_TYPE)
# when
archive.seek(0)
response = self.client.post(
url,
format='multipart',
data={
'archive': archive,
},
# + headers
HTTP_IN_PROGRESS='false',
HTTP_SLUG='external-id')
# then
self.assertEqual(response.status_code,
status.HTTP_415_UNSUPPORTED_MEDIA_TYPE)
diff --git a/swh/deposit/tests/common.py b/swh/deposit/tests/common.py
index ad3aafed..fa5faf36 100644
--- a/swh/deposit/tests/common.py
+++ b/swh/deposit/tests/common.py
@@ -1,165 +1,166 @@
# Copyright (C) 2017 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import base64
import hashlib
import os
import shutil
from django.core.urlresolvers import reverse
from django.test import TestCase
from io import BytesIO
from rest_framework import status
from swh.deposit.config import COL_IRI, EM_IRI
from swh.deposit.models import DepositClient, DepositCollection
from swh.deposit.models import DepositRequestType
from swh.deposit.parsers import parse_xml
from swh.deposit.settings.testing import MEDIA_ROOT
class BasicTestCase(TestCase):
"""Mixin intended for data setup purposes (user, collection, etc...)
"""
def setUp(self):
"""Define the test client and other test variables."""
super().setUp()
# expanding diffs in tests
self.maxDiff = None
# basic minimum test data
deposit_request_types = {}
# Add deposit request types
for deposit_request_type in ['archive', 'metadata']:
drt = DepositRequestType(name=deposit_request_type)
drt.save()
deposit_request_types[deposit_request_type] = drt
_name = 'hal'
# set collection up
_collection = DepositCollection(name=_name)
_collection.save()
# set user/client up
_client = DepositClient.objects.create_user(username=_name,
password=_name)
_client.collections = [_collection.id]
_client.save()
self.collection = _collection
self.user = _client
self.username = _name
self.userpass = _name
self.deposit_request_types = deposit_request_types
def tearDown(self):
# Clean up uploaded files in temporary directory (tests have
# their own media root folder)
if os.path.exists(MEDIA_ROOT):
for d in os.listdir(MEDIA_ROOT):
shutil.rmtree(os.path.join(MEDIA_ROOT, d))
class WithAuthTestCase(TestCase):
"""Mixin intended for testing the api with basic authentication.
"""
def setUp(self):
super().setUp()
_token = '%s:%s' % (self.username, self.userpass)
token = base64.b64encode(_token.encode('utf-8'))
authorization = 'Basic %s' % token.decode('utf-8')
self.client.credentials(HTTP_AUTHORIZATION=authorization)
def tearDown(self):
super().tearDown()
self.client.credentials()
class CommonCreationRoutine(TestCase):
"""Mixin class to share initialization routine.
cf:
`class`:test_deposit_update.DepositReplaceExistingDataTest
`class`:test_deposit_update.DepositUpdateDepositWithNewDataTest
`class`:test_deposit_update.DepositUpdateFailuresTest
`class`:test_deposit_delete.DepositDeleteTest
"""
def setUp(self):
super().setUp()
self.atom_entry_data0 = b"""
some-external-id
"""
def create_simple_deposit_partial(self):
"""Create a simple deposit (1 request) in `partial` state and returns
its new identifier.
Returns:
deposit id
"""
response = self.client.post(
reverse(COL_IRI, args=[self.username]),
content_type='application/atom+xml;type=entry',
data=self.atom_entry_data0,
+ HTTP_SLUG='external-id',
HTTP_IN_PROGRESS='true')
# then
assert response.status_code == status.HTTP_201_CREATED
response_content = parse_xml(BytesIO(response.content))
deposit_id = response_content[
'{http://www.w3.org/2005/Atom}deposit_id']
return deposit_id
def _update_deposit_with_status(self, deposit_id, status_partial=False):
"""Add to a given deposit another archive and update its current
status to `ready` (by default).
Returns:
deposit id
"""
# add an archive
data_text = b'some content'
md5sum = hashlib.md5(data_text).hexdigest()
# when
response = self.client.post(
reverse(EM_IRI, args=[self.username, deposit_id]),
content_type='application/zip', # as zip
data=data_text,
# + headers
HTTP_CONTENT_MD5=md5sum,
HTTP_PACKAGING='http://purl.org/net/sword/package/SimpleZip',
HTTP_IN_PROGRESS=status_partial,
HTTP_CONTENT_LENGTH=len(data_text),
HTTP_CONTENT_DISPOSITION='attachment; filename=filename0')
# then
assert response.status_code == status.HTTP_201_CREATED
return deposit_id
def create_deposit_ready(self):
"""Create a complex deposit (2 requests) in status `ready`.
"""
deposit_id = self.create_simple_deposit_partial()
deposit_id = self._update_deposit_with_status(deposit_id)
return deposit_id
def create_deposit_partial(self):
"""Create a complex deposit (2 requests) in status `partial`.
"""
deposit_id = self.create_simple_deposit_partial()
deposit_id = self._update_deposit_with_status(
deposit_id, status_partial=True)
return deposit_id