diff --git a/requirements.txt b/requirements-server.txt copy from requirements.txt copy to requirements-server.txt --- a/requirements.txt +++ b/requirements-server.txt @@ -1,5 +1,2 @@ -vcversioner -click Django < 2.0 djangorestframework -xmltodict diff --git a/requirements-swh.txt b/requirements-swh-server.txt copy from requirements-swh.txt copy to requirements-swh-server.txt --- a/requirements-swh.txt +++ b/requirements-swh-server.txt @@ -1,4 +1,3 @@ -swh.core >= 0.0.36 swh.loader.tar >= 0.0.39 swh.loader.core >= 0.0.32 swh.scheduler >= 0.0.39 diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,5 +1 @@ swh.core >= 0.0.36 -swh.loader.tar >= 0.0.39 -swh.loader.core >= 0.0.32 -swh.scheduler >= 0.0.39 -swh.model >= 0.0.26 diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ vcversioner click -Django < 2.0 -djangorestframework xmltodict diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -47,7 +47,9 @@ install_requires=parse_requirements() + parse_requirements('swh'), tests_require=parse_requirements('test'), setup_requires=['vcversioner'], - extras_require={'testing': parse_requirements('test')}, + extras_require={'testing': parse_requirements('test'), + 'server': (parse_requirements('server') + + parse_requirements('swh-server'))}, vcversioner={}, include_package_data=True, entry_points=''' diff --git a/swh/deposit/api/common.py b/swh/deposit/api/common.py --- a/swh/deposit/api/common.py +++ b/swh/deposit/api/common.py @@ -21,7 +21,7 @@ SWHDefaultConfig, EDIT_SE_IRI, EM_IRI, CONT_FILE_IRI, ARCHIVE_KEY, METADATA_KEY, RAW_METADATA_KEY, STATE_IRI, DEPOSIT_STATUS_DEPOSITED, DEPOSIT_STATUS_PARTIAL, - DEPOSIT_STATUS_LOAD_SUCCESS + DEPOSIT_STATUS_LOAD_SUCCESS, ARCHIVE_TYPE, METADATA_TYPE ) from ..errors import ( MAX_UPLOAD_SIZE_EXCEEDED, BAD_REQUEST, ERROR_CONTENT, @@ -30,7 +30,7 @@ NOT_FOUND, make_error_response, METHOD_NOT_ALLOWED ) from ..models import ( - Deposit, DepositRequest, DepositCollection, DepositRequestType, + Deposit, DepositRequest, DepositCollection, DepositClient ) from ..parsers import parse_xml @@ -62,12 +62,6 @@ """Base deposit request class sharing multiple common behaviors. """ - def __init__(self): - super().__init__() - deposit_request_types = DepositRequestType.objects.all() - self.deposit_request_types = { - type.name: type for type in deposit_request_types - } def _read_headers(self, req): """Read and unify the necessary headers from the request (those are @@ -205,19 +199,19 @@ if replace_metadata: DepositRequest.objects.filter( deposit=deposit, - type=self.deposit_request_types[METADATA_KEY]).delete() + type=METADATA_TYPE).delete() if replace_archives: DepositRequest.objects.filter( deposit=deposit, - type=self.deposit_request_types[ARCHIVE_KEY]).delete() + type=ARCHIVE_TYPE).delete() deposit_request = None archive_file = deposit_request_data.get(ARCHIVE_KEY) if archive_file: deposit_request = DepositRequest( - type=self.deposit_request_types[ARCHIVE_KEY], + type=ARCHIVE_TYPE, deposit=deposit, archive=archive_file) deposit_request.save() @@ -226,7 +220,7 @@ if metadata: raw_metadata = deposit_request_data.get(RAW_METADATA_KEY) deposit_request = DepositRequest( - type=self.deposit_request_types[METADATA_KEY], + type=METADATA_TYPE, deposit=deposit, metadata=metadata, raw_metadata=raw_metadata) @@ -246,7 +240,7 @@ 'The deposit %s does not exist' % deposit_id) DepositRequest.objects.filter( deposit=deposit, - type=self.deposit_request_types[ARCHIVE_KEY]).delete() + type=ARCHIVE_TYPE).delete() return {} diff --git a/swh/deposit/api/private/__init__.py b/swh/deposit/api/private/__init__.py --- a/swh/deposit/api/private/__init__.py +++ b/swh/deposit/api/private/__init__.py @@ -19,7 +19,7 @@ Args: deposit (Deposit): Deposit to list requests for - request_type (str): Archive or metadata type + request_type (str): 'archive' or 'metadata' Yields: deposit requests of type request_type associated to the deposit @@ -29,7 +29,7 @@ deposit = Deposit.objects.get(pk=deposit) deposit_requests = DepositRequest.objects.filter( - type=self.deposit_request_types[request_type], + type=request_type, deposit=deposit).order_by('id') for deposit_request in deposit_requests: diff --git a/swh/deposit/cli/__init__.py b/swh/deposit/cli/__init__.py new file mode 100644 --- /dev/null +++ b/swh/deposit/cli/__init__.py @@ -0,0 +1,37 @@ +# Copyright (C) 2017-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import click +import logging + +logger = logging.getLogger(__name__) + + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) + + +@click.group(context_settings=CONTEXT_SETTINGS) +@click.option('--log-level', '-l', default='INFO', + type=click.Choice(logging._nameToLevel.keys()), + help="Log level (default to INFO)") +@click.pass_context +def cli(ctx, log_level): + logger.setLevel(log_level) + ctx.ensure_object(dict) + + +def main(): + logging.basicConfig() + from . import deposit # noqa + try: + from . import admin # noqa + except ImportError: # server part is optional + pass + + return cli(auto_envvar_prefix='SWH_DEPOSIT') + + +if __name__ == '__main__': + main() diff --git a/swh/deposit/cli/admin.py b/swh/deposit/cli/admin.py new file mode 100644 --- /dev/null +++ b/swh/deposit/cli/admin.py @@ -0,0 +1,178 @@ +# Copyright (C) 2017-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import click + +from swh.deposit.config import setup_django_for +from swh.deposit.cli import cli + + +@cli.group('admin') +@click.option('--config-file', '-C', default=None, + type=click.Path(exists=True, dir_okay=False,), + help="Optional extra configuration file.") +@click.option('--platform', default='development', + type=click.Choice(['development', 'production']), + help='development or production platform') +@click.pass_context +def admin(ctx, config_file, platform): + """Server administration tasks (manipulate user or collections)""" + # configuration happens here + setup_django_for(platform, config_file=config_file) + + +@admin.group('user') +@click.pass_context +def user(ctx): + """Manipulate user.""" + # configuration happens here + pass + + +def _create_collection(name): + """Create the collection with name if it does not exist. + + Args: + name (str): collection's name + + Returns: + collection (DepositCollection): the existing collection object + (created or not) + + """ + # to avoid loading too early django namespaces + from swh.deposit.models import DepositCollection + + try: + collection = DepositCollection.objects.get(name=name) + click.echo('Collection %s exists, nothing to do.' % name) + except DepositCollection.DoesNotExist: + click.echo('Create new collection %s' % name) + collection = DepositCollection.objects.create(name=name) + click.echo('Collection %s created' % name) + return collection + + +@user.command('create') +@click.option('--username', required=True, help="User's name") +@click.option('--password', required=True, + help="Desired user's password (plain).") +@click.option('--firstname', default='', help="User's first name") +@click.option('--lastname', default='', help="User's last name") +@click.option('--email', default='', help="User's email") +@click.option('--collection', help="User's collection") +@click.option('--provider-url', default='', help="Provider URL") +@click.option('--domain', help="The domain") +@click.pass_context +def user_create(ctx, username, password, firstname, lastname, email, + collection, provider_url, domain): + """Create a user with some needed information (password, collection) + + If the collection does not exist, the collection is then created + alongside. + + The password is stored encrypted using django's utilies. + + """ + # to avoid loading too early django namespaces + from swh.deposit.models import DepositClient + + # If collection is not provided, fallback to username + if not collection: + collection = username + click.echo('collection: %s' % collection) + # create the collection if it does not exist + collection = _create_collection(collection) + + # user create/update + try: + user = DepositClient.objects.get(username=username) + click.echo('User %s exists, updating information.' % user) + user.set_password(password) + except DepositClient.DoesNotExist: + click.echo('Create new user %s' % username) + user = DepositClient.objects.create_user( + username=username, + password=password) + + user.collections = [collection.id] + user.first_name = firstname + user.last_name = lastname + user.email = email + user.is_active = True + user.provider_url = provider_url + user.domain = domain + user.save() + + click.echo('Information registered for user %s' % user) + + +@user.command('list') +@click.pass_context +def user_list(ctx): + """List existing users. + + This entrypoint is not paginated yet as there is not a lot of + entry. + + """ + # to avoid loading too early django namespaces + from swh.deposit.models import DepositClient + users = DepositClient.objects.all() + if not users: + output = 'Empty user list' + else: + output = '\n'.join((user.username for user in users)) + click.echo(output) + + +@user.command('exists') +@click.argument('username', required=True) +@click.pass_context +def user_exists(ctx, username): + """Check if user exists. + """ + # to avoid loading too early django namespaces + from swh.deposit.models import DepositClient + try: + DepositClient.objects.get(username=username) + click.echo('User %s exists.' % username) + ctx.exit(0) + except DepositClient.DoesNotExist: + click.echo('User %s does not exists.' % username) + ctx.exit(1) + + +@admin.group('collection') +@click.pass_context +def collection(ctx): + """Manipulate collections.""" + pass + + +@collection.command('create') +@click.option('--name', required=True, help="Collection's name") +@click.pass_context +def collection_create(ctx, name): + _create_collection(name) + + +@collection.command('list') +@click.pass_context +def collection_list(ctx): + """List existing collections. + + This entrypoint is not paginated yet as there is not a lot of + entry. + + """ + # to avoid loading too early django namespaces + from swh.deposit.models import DepositCollection + collections = DepositCollection.objects.all() + if not collections: + output = 'Empty collection list' + else: + output = '\n'.join((col.name for col in collections)) + click.echo(output) diff --git a/swh/deposit/cli.py b/swh/deposit/cli/deposit.py rename from swh/deposit/cli.py rename to swh/deposit/cli/deposit.py --- a/swh/deposit/cli.py +++ b/swh/deposit/cli/deposit.py @@ -3,174 +3,17 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import click -import os import logging +import os import uuid -from swh.deposit.config import setup_django_for -try: - from swh.deposit.client import PublicApiDepositClient -except ImportError: - logging.warn("Optional client subcommand unavailable. " - "Install swh.deposit.client to be able to use it.") - - -CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) - - -@click.group(context_settings=CONTEXT_SETTINGS) -@click.option('--config-file', '-C', default=None, - type=click.Path(exists=True, dir_okay=False,), - help="Optional extra configuration file.") -@click.option('--platform', default='development', - type=click.Choice(['development', 'production']), - help='development or production platform') -@click.option('--verbose/--no-verbose', default=False, - help='Verbose mode') -@click.pass_context -def cli(ctx, config_file, platform, verbose): - logger = logging.getLogger(__name__) - logger.addHandler(logging.StreamHandler()) - _loglevel = logging.DEBUG if verbose else logging.INFO - logger.setLevel(_loglevel) - - ctx.ensure_object(dict) - - # configuration happens here - setup_django_for(platform, config_file=config_file) - - ctx.obj = {'loglevel': _loglevel} - - -@cli.group('user') -@click.pass_context -def user(ctx): - """Manipulate user.""" - pass - - -def _create_collection(name): - """Create the collection with name if it does not exist. - - Args: - name (str): collection's name - - Returns: - collection (DepositCollection): the existing collection object - (created or not) - - """ - # to avoid loading too early django namespaces - from swh.deposit.models import DepositCollection - - try: - collection = DepositCollection.objects.get(name=name) - click.echo('Collection %s exists, nothing to do.' % name) - except DepositCollection.DoesNotExist: - click.echo('Create new collection %s' % name) - collection = DepositCollection.objects.create(name=name) - click.echo('Collection %s created' % name) - return collection - - -@user.command('create') -@click.option('--username', required=True, help="User's name") -@click.option('--password', required=True, - help="Desired user's password (plain).") -@click.option('--firstname', default='', help="User's first name") -@click.option('--lastname', default='', help="User's last name") -@click.option('--email', default='', help="User's email") -@click.option('--collection', help="User's collection") -@click.pass_context -def user_create(ctx, username, password, firstname, lastname, email, - collection): - """Create a user with some needed information (password, collection) - - If the collection does not exist, the collection is then created - alongside. - - The password is stored encrypted using django's utilies. - - """ - # to avoid loading too early django namespaces - from swh.deposit.models import DepositClient - - click.echo('collection: %s' % collection) - # create the collection if it does not exist - collection = _create_collection(collection) - - # user create/update - try: - user = DepositClient.objects.get(username=username) - click.echo('User %s exists, updating information.' % user) - user.set_password(password) - except DepositClient.DoesNotExist: - click.echo('Create new user %s' % username) - user = DepositClient.objects.create_user( - username=username, - password=password) - - user.collections = [collection.id] - user.first_name = firstname - user.last_name = lastname - user.email = email - user.is_active = True - user.save() - - click.echo('Information registered for user %s' % user) - - -@user.command('list') -@click.pass_context -def user_list(ctx): - """List existing users. - - This entrypoint is not paginated yet as there is not a lot of - entry. - - """ - # to avoid loading too early django namespaces - from swh.deposit.models import DepositClient - users = DepositClient.objects.all() - if not users: - output = 'Empty user list' - else: - output = '\n'.join((user.username for user in users)) - click.echo(output) - - -@cli.group('collection') -@click.pass_context -def collection(ctx): - """Manipulate collection.""" - pass - - -@collection.command('create') -@click.option('--name', required=True, help="Collection's name") -@click.pass_context -def collection_create(ctx, name): - _create_collection(name) - +import click -@collection.command('list') -@click.pass_context -def collection_list(ctx): - """List existing collections. +from swh.deposit.client import PublicApiDepositClient +from swh.deposit.cli import cli - This entrypoint is not paginated yet as there is not a lot of - entry. - """ - # to avoid loading too early django namespaces - from swh.deposit.models import DepositCollection - collections = DepositCollection.objects.all() - if not collections: - output = 'Empty collection list' - else: - output = '\n'.join((col.name for col in collections)) - click.echo(output) +logger = logging.getLogger(__name__) class InputError(ValueError): @@ -287,7 +130,8 @@ if 'error' in sd_content: raise InputError('Service document retrieval: %s' % ( sd_content['error'], )) - collection = sd_content['collection'] + collection = sd_content[ + 'service']['workspace']['collection']['sword:name'] if not slug: # generate slug @@ -300,7 +144,7 @@ 'metadata': metadata, 'collection': collection, 'slug': slug, - 'partial': partial, + 'in_progress': partial, 'client': client, 'url': url, 'deposit_id': deposit_id, @@ -308,55 +152,42 @@ } -def deposit_status(config, dry_run, logger): +def _subdict(d, keys): + 'return a dict from d with only given keys' + return {k: v for k, v in d.items() if k in keys} + + +def deposit_status(config, logger): logger.debug('Status deposit') + keys = ('collection', 'deposit_id') client = config['client'] - collection = config['collection'] - deposit_id = config['deposit_id'] - if not dry_run: - r = client.deposit_status(collection, deposit_id, logger) - return r - return {} + return client.deposit_status( + **_subdict(config, keys)) -def deposit_create(config, dry_run, logger): +def deposit_create(config, logger): """Delegate the actual deposit to the deposit client. """ logger.debug('Create deposit') client = config['client'] - collection = config['collection'] - archive_path = config['archive'] - metadata_path = config['metadata'] - slug = config['slug'] - in_progress = config['partial'] - if not dry_run: - r = client.deposit_create(collection, slug, archive_path, - metadata_path, in_progress, logger) - return r - return {} - - -def deposit_update(config, dry_run, logger): + keys = ('collection', 'archive', 'metadata', 'slug', 'in_progress') + return client.deposit_create( + **_subdict(config, keys)) + + +def deposit_update(config, logger): """Delegate the actual deposit to the deposit client. """ logger.debug('Update deposit') client = config['client'] - collection = config['collection'] - deposit_id = config['deposit_id'] - archive_path = config['archive'] - metadata_path = config['metadata'] - slug = config['slug'] - in_progress = config['partial'] - replace = config['replace'] - if not dry_run: - r = client.deposit_update(collection, deposit_id, slug, archive_path, - metadata_path, in_progress, replace, logger) - return r - return {} + keys = ('collection', 'deposit_id', 'archive', 'metadata', + 'slug', 'in_progress', 'replace') + return client.deposit_update( + **_subdict(config, keys)) @cli.command() @@ -386,8 +217,6 @@ help="(Optional) Deposit server api endpoint. By default, https://deposit.softwareheritage.org/1") # noqa @click.option('--status/--no-status', default=False, help="(Optional) Deposit's status") -@click.option('--dry-run/--no-dry-run', default=False, - help='(Optional) No-op deposit') @click.option('--verbose/--no-verbose', default=False, help='Verbose mode') @click.pass_context @@ -396,7 +225,7 @@ archive_deposit=False, metadata_deposit=False, collection=None, slug=None, partial=False, deposit_id=None, replace=False, status=False, - url='https://deposit.softwareheritage.org/1', dry_run=True, + url='https://deposit.softwareheritage.org/1', verbose=False): """Software Heritage Public Deposit Client @@ -407,11 +236,6 @@ https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html. """ - logger = logging.getLogger(__name__) - - if dry_run: - logger.info("**DRY RUN**") - config = {} try: @@ -436,18 +260,10 @@ deposit_id = config['deposit_id'] if status and deposit_id: - r = deposit_status(config, dry_run, logger) + r = deposit_status(config, logger) elif not status and deposit_id: - r = deposit_update(config, dry_run, logger) + r = deposit_update(config, logger) elif not status and not deposit_id: - r = deposit_create(config, dry_run, logger) + r = deposit_create(config, logger) logger.info(r) - - -def main(): - return cli(auto_envvar_prefix='SWH_DEPOSIT') - - -if __name__ == '__main__': - main() diff --git a/swh/deposit/client/__init__.py b/swh/deposit/client/__init__.py --- a/swh/deposit/client/__init__.py +++ b/swh/deposit/client/__init__.py @@ -11,12 +11,16 @@ import os import requests import xmltodict +import logging from abc import ABCMeta, abstractmethod from swh.core.config import SWHConfig +logger = logging.getLogger(__name__) + + def _parse(stream, encoding='utf-8'): """Given a xml stream, parse the result. @@ -115,14 +119,14 @@ - update a given deposit's status """ - def archive_get(self, archive_update_url, archive_path, log=None): + def archive_get(self, archive_update_url, archive): """Retrieve the archive from the deposit to a local directory. Args: archive_update_url (str): The full deposit archive(s)'s raw content to retrieve locally - archive_path (str): the local archive's path where to store + archive (str): the local archive's path where to store the raw content Returns: @@ -132,20 +136,19 @@ """ r = self.do('get', archive_update_url, stream=True) if r.ok: - with open(archive_path, 'wb') as f: + with open(archive, 'wb') as f: for chunk in r.iter_content(): f.write(chunk) - return archive_path + return archive msg = 'Problem when retrieving deposit archive at %s' % ( archive_update_url, ) - if log: - log.error(msg) + logger.error(msg) raise ValueError(msg) - def metadata_get(self, metadata_url, log=None): + def metadata_get(self, metadata_url): """Retrieve the metadata information on a given deposit. Args: @@ -162,8 +165,7 @@ return r.json() msg = 'Problem when retrieving metadata at %s' % metadata_url - if log: - log.error(msg) + logger.error(msg) raise ValueError(msg) @@ -189,7 +191,7 @@ self.do('put', update_status_url, json=payload) - def check(self, check_url, log=None): + def check(self, check_url): """Check the deposit's associated data (metadata, archive(s)) Args: @@ -202,8 +204,7 @@ return data['status'] msg = 'Problem when checking deposit %s' % check_url - if log: - log.error(msg) + logger.error(msg) raise ValueError(msg) @@ -316,7 +317,7 @@ """Parse service document's success response. """ - return _parse_with_filter(xml_content, keys=['collection']) + return _parse(xml_content) class StatusDepositClient(BaseDepositClient): @@ -490,12 +491,12 @@ return files, headers - def compute_information(self, collection, archive_path, metadata_path, + def compute_information(self, collection, archive, metadata, in_progress, slug, **kwargs): info = self._compute_information( - collection, archive_path, in_progress, slug) + collection, archive, in_progress, slug) info_meta = self._compute_information( - collection, metadata_path, in_progress, slug, is_archive=False) + collection, metadata, in_progress, slug, is_archive=False) files, headers = self._multipart_info(info, info_meta) return {'files': files, 'headers': headers} @@ -515,35 +516,35 @@ class PublicApiDepositClient(BaseApiDepositClient): """Public api deposit client.""" - def service_document(self, log=None): + def service_document(self): """Retrieve service document endpoint's information.""" return ServiceDocumentDepositClient(self.config).execute() - def deposit_status(self, collection, deposit_id, log=None): + def deposit_status(self, collection, deposit_id): """Retrieve status information on a deposit.""" return StatusDepositClient(self.config).execute( collection, deposit_id) - def deposit_create(self, collection, slug, archive_path=None, - metadata_path=None, in_progress=False, log=None): + def deposit_create(self, collection, slug, archive=None, + metadata=None, in_progress=False): """Create a new deposit (archive, metadata, both as multipart).""" - if archive_path and not metadata_path: + if archive and not metadata: return CreateArchiveDepositClient(self.config).execute( - collection, archive_path, in_progress, slug) - elif not archive_path and metadata_path: + collection, archive, in_progress, slug) + elif not archive and metadata: return CreateMetadataDepositClient(self.config).execute( - collection, metadata_path, in_progress, slug, + collection, metadata, in_progress, slug, is_archive=False) else: return CreateMultipartDepositClient(self.config).execute( - collection, archive_path, metadata_path, in_progress, + collection, archive, metadata, in_progress, slug) - def deposit_update(self, collection, deposit_id, slug, archive_path=None, - metadata_path=None, in_progress=False, - replace=False, log=None): + def deposit_update(self, collection, deposit_id, slug, archive=None, + metadata=None, in_progress=False, + replace=False): """Update (add/replace) existing deposit (archive, metadata, both).""" - r = self.deposit_status(collection, deposit_id, log=log) + r = self.deposit_status(collection, deposit_id) if 'error' in r: return r @@ -556,19 +557,19 @@ 'deposit_status': status, 'deposit_id': deposit_id, } - if archive_path and not metadata_path: + if archive and not metadata: r = UpdateArchiveDepositClient(self.config).execute( - collection, archive_path, in_progress, slug, - deposit_id=deposit_id, replace=replace, log=log) - elif not archive_path and metadata_path: + collection, archive, in_progress, slug, + deposit_id=deposit_id, replace=replace) + elif not archive and metadata: r = UpdateMetadataDepositClient(self.config).execute( - collection, metadata_path, in_progress, slug, - deposit_id=deposit_id, replace=replace, log=log) + collection, metadata, in_progress, slug, + deposit_id=deposit_id, replace=replace) else: r = UpdateMultipartDepositClient(self.config).execute( - collection, archive_path, metadata_path, in_progress, - slug, deposit_id=deposit_id, replace=replace, log=log) + collection, archive, metadata, in_progress, + slug, deposit_id=deposit_id, replace=replace) if 'error' in r: return r - return self.deposit_status(collection, deposit_id, log=log) + return self.deposit_status(collection, deposit_id) diff --git a/swh/deposit/config.py b/swh/deposit/config.py --- a/swh/deposit/config.py +++ b/swh/deposit/config.py @@ -29,6 +29,7 @@ ARCHIVE_TYPE = 'archive' METADATA_TYPE = 'metadata' + AUTHORIZED_PLATFORMS = ['development', 'production', 'testing'] DEPOSIT_STATUS_REJECTED = 'rejected' @@ -46,7 +47,7 @@ } -def setup_django_for(platform, config_file=None): +def setup_django_for(platform=None, config_file=None): """Setup function for command line tools (swh.deposit.create_user) to initialize the needed db access. @@ -64,11 +65,13 @@ ValueError in case of wrong platform inputs. """ - if platform not in AUTHORIZED_PLATFORMS: - raise ValueError('Platform should be one of %s' % AUTHORIZED_PLATFORMS) - - os.environ.setdefault('DJANGO_SETTINGS_MODULE', - 'swh.deposit.settings.%s' % platform) + if platform is not None: + if platform not in AUTHORIZED_PLATFORMS: + raise ValueError('Platform should be one of %s' % + AUTHORIZED_PLATFORMS) + if 'DJANGO_SETTINGS_MODULE' not in os.environ: + os.environ['DJANGO_SETTINGS_MODULE'] = ( + 'swh.deposit.settings.%s' % platform) if config_file: os.environ.setdefault('SWH_CONFIG_FILENAME', config_file) diff --git a/swh/deposit/loader/loader.py b/swh/deposit/loader/loader.py --- a/swh/deposit/loader/loader.py +++ b/swh/deposit/loader/loader.py @@ -49,7 +49,7 @@ def prepare_origin_visit(self, *, deposit_meta_url, **kwargs): self.metadata = self.deposit_client.metadata_get( - deposit_meta_url, log=self.log) + deposit_meta_url) self.origin = self.metadata['origin'] self.visit_date = None @@ -65,7 +65,7 @@ self.temporary_directory = temporary_directory archive_path = os.path.join(temporary_directory.name, 'archive.zip') archive = self.deposit_client.archive_get( - archive_url, archive_path, log=self.log) + archive_url, archive_path) metadata = self.metadata revision = metadata['revision'] diff --git a/swh/deposit/migrations/0015_auto_20190411_1421.py b/swh/deposit/migrations/0015_auto_20190411_1421.py new file mode 100644 --- /dev/null +++ b/swh/deposit/migrations/0015_auto_20190411_1421.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2019-04-11 14:21 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('deposit', '0014_auto_20180720_1221'), + ] + + operations = [ + migrations.AlterField( + model_name='depositrequest', + name='type', + field=models.CharField(choices=[ + ('archive', 'archive'), + ('metadata', 'metadata')], max_length=8), + ), + migrations.DeleteModel( + name='DepositRequestType', + ), + ] diff --git a/swh/deposit/models.py b/swh/deposit/models.py --- a/swh/deposit/models.py +++ b/swh/deposit/models.py @@ -16,7 +16,7 @@ from .config import ( DEPOSIT_STATUS_VERIFIED, DEPOSIT_STATUS_DEPOSITED, DEPOSIT_STATUS_PARTIAL, DEPOSIT_STATUS_LOAD_SUCCESS, DEPOSIT_STATUS_LOAD_FAILURE, - DEPOSIT_STATUS_REJECTED + DEPOSIT_STATUS_REJECTED, ARCHIVE_TYPE, METADATA_TYPE ) @@ -141,20 +141,6 @@ return str(d) -class DepositRequestType(models.Model): - """Deposit request type made by clients (either archive or metadata) - - """ - id = models.BigAutoField(primary_key=True) - name = models.TextField() - - class Meta: - db_table = 'deposit_request_type' - - def __str__(self): - return str({'id': self.id, 'name': self.name}) - - def client_directory_path(instance, filename): """Callable to upload archive in MEDIA_ROOT/user_/ @@ -170,6 +156,10 @@ return 'client_{0}/{1}'.format(instance.deposit.client.id, filename) +REQUEST_TYPES = [(ARCHIVE_TYPE, ARCHIVE_TYPE), + (METADATA_TYPE, METADATA_TYPE)] + + class DepositRequest(models.Model): """Deposit request associated to one deposit. @@ -185,8 +175,7 @@ # this can be null when type is 'metadata' archive = models.FileField(null=True, upload_to=client_directory_path) - type = models.ForeignKey( - 'DepositRequestType', models.DO_NOTHING) + type = models.CharField(max_length=8, choices=REQUEST_TYPES) class Meta: db_table = 'deposit_request' diff --git a/swh/deposit/settings/production.py b/swh/deposit/settings/production.py --- a/swh/deposit/settings/production.py +++ b/swh/deposit/settings/production.py @@ -45,6 +45,8 @@ "Production: invalid configuration; missing %s config entry." % ( key, )) +ALLOWED_HOSTS += conf.get('allowed_hosts', []) + private_conf = conf['private'] SECRET_KEY = private_conf['secret_key'] diff --git a/swh/deposit/tests/api/test_deposit_binary.py b/swh/deposit/tests/api/test_deposit_binary.py --- a/swh/deposit/tests/api/test_deposit_binary.py +++ b/swh/deposit/tests/api/test_deposit_binary.py @@ -447,7 +447,7 @@ deposit_request = DepositRequest.objects.get(deposit=deposit) self.assertEqual(deposit_request.deposit, deposit) - self.assertEqual(deposit_request.type.name, 'archive') + self.assertEqual(deposit_request.type, 'archive') self.assertRegex(deposit_request.archive.name, self.archive['name']) # 2nd archive to upload @@ -486,12 +486,12 @@ # 2 deposit requests for the same deposit self.assertEqual(len(deposit_requests), 2) self.assertEqual(deposit_requests[0].deposit, deposit) - self.assertEqual(deposit_requests[0].type.name, 'archive') + self.assertEqual(deposit_requests[0].type, 'archive') self.assertRegex(deposit_requests[0].archive.name, self.archive['name']) self.assertEqual(deposit_requests[1].deposit, deposit) - self.assertEqual(deposit_requests[1].type.name, 'archive') + self.assertEqual(deposit_requests[1].type, 'archive') self.assertRegex(deposit_requests[1].archive.name, archive2['name']) diff --git a/swh/deposit/tests/api/test_deposit_delete.py b/swh/deposit/tests/api/test_deposit_delete.py --- a/swh/deposit/tests/api/test_deposit_delete.py +++ b/swh/deposit/tests/api/test_deposit_delete.py @@ -29,9 +29,9 @@ self.assertEqual(len(deposit_requests), 2) for dr in deposit_requests: - if dr.type.name == ARCHIVE_KEY: + if dr.type == ARCHIVE_KEY: continue - elif dr.type.name == METADATA_KEY: + elif dr.type == METADATA_KEY: continue else: self.fail('only archive and metadata type should exist ' @@ -47,8 +47,8 @@ requests = list(DepositRequest.objects.filter(deposit=deposit)) self.assertEqual(len(requests), 2) - self.assertEqual(requests[0].type.name, 'metadata') - self.assertEqual(requests[1].type.name, 'metadata') + self.assertEqual(requests[0].type, 'metadata') + self.assertEqual(requests[1].type, 'metadata') def test_delete_archive_on_undefined_deposit_fails(self): """Delete undefined deposit returns a 404 response diff --git a/swh/deposit/tests/api/test_deposit_multipart.py b/swh/deposit/tests/api/test_deposit_multipart.py --- a/swh/deposit/tests/api/test_deposit_multipart.py +++ b/swh/deposit/tests/api/test_deposit_multipart.py @@ -157,7 +157,7 @@ self.assertEqual(len(deposit_requests), 2) for deposit_request in deposit_requests: self.assertEqual(deposit_request.deposit, deposit) - if deposit_request.type.name == 'archive': + if deposit_request.type == 'archive': self.assertRegex(deposit_request.archive.name, self.archive['name']) self.assertIsNone(deposit_request.metadata) @@ -226,7 +226,7 @@ self.assertEqual(len(deposit_requests), 2) for deposit_request in deposit_requests: self.assertEqual(deposit_request.deposit, deposit) - if deposit_request.type.name == 'archive': + if deposit_request.type == 'archive': self.assertRegex(deposit_request.archive.name, self.archive['name']) self.assertIsNone(deposit_request.metadata) @@ -296,7 +296,7 @@ self.assertEqual(len(deposit_requests), 2) for deposit_request in deposit_requests: self.assertEqual(deposit_request.deposit, deposit) - if deposit_request.type.name == 'archive': + if deposit_request.type == 'archive': self.assertRegex(deposit_request.archive.name, self.archive['name']) else: @@ -327,7 +327,7 @@ self.assertEqual(len(deposit_requests), 2) for deposit_request in deposit_requests: self.assertEqual(deposit_request.deposit, deposit) - if deposit_request.type.name == 'archive': + if deposit_request.type == 'archive': self.assertRegex(deposit_request.archive.name, self.archive['name']) else: diff --git a/swh/deposit/tests/api/test_deposit_update.py b/swh/deposit/tests/api/test_deposit_update.py --- a/swh/deposit/tests/api/test_deposit_update.py +++ b/swh/deposit/tests/api/test_deposit_update.py @@ -46,21 +46,21 @@ deposit = Deposit.objects.get(pk=deposit_id) requests = DepositRequest.objects.filter( deposit=deposit, - type=self.deposit_request_types['archive']) + type='archive') assert len(list(requests)) == 1 assert self.archive['name'] in requests[0].archive.name # we have no metadata for that deposit requests = list(DepositRequest.objects.filter( - deposit=deposit, type=self.deposit_request_types['metadata'])) + deposit=deposit, type='metadata')) assert len(requests) == 0 deposit_id = self._update_deposit_with_status(deposit_id, status_partial=True) requests = list(DepositRequest.objects.filter( - deposit=deposit, type=self.deposit_request_types['metadata'])) + deposit=deposit, type='metadata')) assert len(requests) == 1 update_uri = reverse(EM_IRI, args=[self.collection.name, deposit_id]) @@ -84,14 +84,14 @@ requests = DepositRequest.objects.filter( deposit=deposit, - type=self.deposit_request_types['archive']) + type='archive') self.assertEqual(len(list(requests)), 1) self.assertRegex(requests[0].archive.name, self.archive2['name']) # check we did not touch the other parts requests = list(DepositRequest.objects.filter( - deposit=deposit, type=self.deposit_request_types['metadata'])) + deposit=deposit, type='metadata')) self.assertEqual(len(requests), 1) def test_replace_metadata_to_deposit_is_possible(self): @@ -104,11 +104,11 @@ deposit = Deposit.objects.get(pk=deposit_id) requests = DepositRequest.objects.filter( deposit=deposit, - type=self.deposit_request_types['metadata']) + type='metadata') assert len(list(requests)) == 0 requests = list(DepositRequest.objects.filter( - deposit=deposit, type=self.deposit_request_types['archive'])) + deposit=deposit, type='archive')) assert len(requests) == 1 update_uri = reverse(EDIT_SE_IRI, args=[self.collection.name, @@ -123,7 +123,7 @@ requests = DepositRequest.objects.filter( deposit=deposit, - type=self.deposit_request_types['metadata']) + type='metadata') self.assertEqual(len(list(requests)), 1) metadata = requests[0].metadata @@ -131,7 +131,7 @@ # check we did not touch the other parts requests = list(DepositRequest.objects.filter( - deposit=deposit, type=self.deposit_request_types['archive'])) + deposit=deposit, type='archive')) self.assertEqual(len(requests), 1) def test_add_archive_to_deposit_is_possible(self): @@ -144,13 +144,13 @@ deposit = Deposit.objects.get(pk=deposit_id) requests = DepositRequest.objects.filter( deposit=deposit, - type=self.deposit_request_types['archive']) + type='archive') assert len(list(requests)) == 1 assert self.archive['name'] in requests[0].archive.name requests = list(DepositRequest.objects.filter( - deposit=deposit, type=self.deposit_request_types['metadata'])) + deposit=deposit, type='metadata')) assert len(requests) == 0 update_uri = reverse(EM_IRI, args=[self.collection.name, deposit_id]) @@ -174,7 +174,7 @@ requests = list(DepositRequest.objects.filter( deposit=deposit, - type=self.deposit_request_types['archive']).order_by('id')) + type='archive').order_by('id')) self.assertEqual(len(requests), 2) # first archive still exists @@ -184,7 +184,7 @@ # check we did not touch the other parts requests = list(DepositRequest.objects.filter( - deposit=deposit, type=self.deposit_request_types['metadata'])) + deposit=deposit, type='metadata')) self.assertEqual(len(requests), 0) def test_add_metadata_to_deposit_is_possible(self): @@ -197,12 +197,12 @@ deposit = Deposit.objects.get(pk=deposit_id) requests = DepositRequest.objects.filter( deposit=deposit, - type=self.deposit_request_types['metadata']) + type='metadata') assert len(list(requests)) == 2 requests = list(DepositRequest.objects.filter( - deposit=deposit, type=self.deposit_request_types['archive'])) + deposit=deposit, type='archive')) assert len(requests) == 0 update_uri = reverse(EDIT_SE_IRI, args=[self.collection.name, @@ -217,7 +217,7 @@ requests = DepositRequest.objects.filter( deposit=deposit, - type=self.deposit_request_types['metadata']).order_by('id') + type='metadata').order_by('id') self.assertEqual(len(list(requests)), 3) # a new one was added @@ -225,7 +225,7 @@ # check we did not touch the other parts requests = list(DepositRequest.objects.filter( - deposit=deposit, type=self.deposit_request_types['archive'])) + deposit=deposit, type='archive')) self.assertEqual(len(requests), 0) diff --git a/swh/deposit/tests/common.py b/swh/deposit/tests/common.py --- a/swh/deposit/tests/common.py +++ b/swh/deposit/tests/common.py @@ -23,7 +23,6 @@ DEPOSIT_STATUS_DEPOSITED) from swh.deposit.models import DepositClient, DepositCollection, Deposit from swh.deposit.models import DepositRequest -from swh.deposit.models import DepositRequestType from swh.deposit.parsers import parse_xml from swh.deposit.settings.testing import MEDIA_ROOT from swh.core import tarball @@ -276,12 +275,6 @@ self.maxDiff = None # basic minimum test data - deposit_request_types = {} - # Add deposit request types - for deposit_request_type in ['archive', 'metadata']: - drt = DepositRequestType(name=deposit_request_type) - drt.save() - deposit_request_types[deposit_request_type] = drt _name = 'hal' _provider_url = 'https://hal-test.archives-ouvertes.fr/' @@ -303,8 +296,6 @@ self.username = _name self.userpass = _name - self.deposit_request_types = deposit_request_types - def tearDown(self): super().tearDown() # Clean up uploaded files in temporary directory (tests have @@ -568,6 +559,6 @@ assert deposit_requests is not [] for dr in deposit_requests: - if dr.type.name == 'metadata': + if dr.type == 'metadata': assert deposit_requests[0].metadata is not {} return deposit_id diff --git a/tox.ini b/tox.ini --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,7 @@ [testenv:py3] deps = .[testing] + .[server] pytest-cov pifpaf pytest-django