diff --git a/requirements-swh-server.txt b/requirements-swh-server.txt index 1b77634b..02920660 100644 --- a/requirements-swh-server.txt +++ b/requirements-swh-server.txt @@ -1,4 +1,5 @@ +swh.core[http] swh.loader.tar >= 0.0.39 swh.loader.core >= 0.0.32 swh.scheduler >= 0.0.39 swh.model >= 0.0.26 diff --git a/requirements-swh.txt b/requirements-swh.txt index 5662ee37..32aa75ec 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1 +1 @@ -swh.core >= 0.0.36 +swh.core >= 0.0.60 diff --git a/requirements.txt b/requirements.txt index 68bb2e26..5c72bcaa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ vcversioner click xmltodict iso8601 +requests diff --git a/setup.py b/setup.py index 2b1163be..a4921dc6 100755 --- a/setup.py +++ b/setup.py @@ -1,71 +1,74 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() -def parse_requirements(name=None): - if name: - reqf = 'requirements-%s.txt' % name - else: - reqf = 'requirements.txt' - +def parse_requirements(*names): requirements = [] - if not path.exists(reqf): - return requirements + for name in names: + if name: + reqf = 'requirements-%s.txt' % name + else: + reqf = 'requirements.txt' + + if not path.exists(reqf): + return requirements - with open(reqf) as f: - for line in f.readlines(): - line = line.strip() - if not line or line.startswith('#'): - continue - requirements.append(line) + with open(reqf) as f: + for line in f.readlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + requirements.append(line) return requirements setup( name='swh.deposit', description='Software Heritage Deposit Server', long_description=long_description, long_description_content_type='text/markdown', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/source/swh-deposit/', packages=find_packages(), - install_requires=parse_requirements() + parse_requirements('swh'), + install_requires=parse_requirements(None, 'swh'), tests_require=parse_requirements('test'), setup_requires=['vcversioner'], - extras_require={'testing': parse_requirements('test'), - 'server': (parse_requirements('server') + - parse_requirements('swh-server'))}, + extras_require={ + 'testing': parse_requirements('test', 'server', 'swh-server'), + 'server': parse_requirements('server', 'swh-server')}, vcversioner={}, include_package_data=True, entry_points=''' [console_scripts] swh-deposit=swh.deposit.cli:main + [swh.cli.subcommands] + deposit=swh.deposit.cli:deposit ''', classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', 'Funding': 'https://www.softwareheritage.org/donate', 'Source': 'https://forge.softwareheritage.org/source/swh-deposit', }, ) diff --git a/swh/deposit/cli/__init__.py b/swh/deposit/cli/__init__.py index f07c4060..8d994985 100644 --- a/swh/deposit/cli/__init__.py +++ b/swh/deposit/cli/__init__.py @@ -1,37 +1,37 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import click import logging -logger = logging.getLogger(__name__) - +from swh.core.cli import CONTEXT_SETTINGS -CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +logger = logging.getLogger(__name__) @click.group(context_settings=CONTEXT_SETTINGS) -@click.option('--log-level', '-l', default='INFO', - type=click.Choice(logging._nameToLevel.keys()), - help="Log level (default to INFO)") @click.pass_context -def cli(ctx, log_level): - logger.setLevel(log_level) +def deposit(ctx): + """Deposit main command + """ + logger.debug('deposit') ctx.ensure_object(dict) def main(): logging.basicConfig() - from . import client # noqa - try: - from . import admin # noqa - except ImportError: # server part is optional - pass - - return cli(auto_envvar_prefix='SWH_DEPOSIT') + return deposit(auto_envvar_prefix='SWH_DEPOSIT') + +# These import statements MUST be executed after defining the 'deposit' group +# since the subcommands in these are defined using this 'deposit' group. +from . import client # noqa +try: + from . import admin # noqa +except ImportError: # server part is optional + logger.debug('admin subcommand not loaded') if __name__ == '__main__': main() diff --git a/swh/deposit/cli/client.py b/swh/deposit/cli/client.py index d8db2592..7a7b8f73 100644 --- a/swh/deposit/cli/client.py +++ b/swh/deposit/cli/client.py @@ -1,353 +1,353 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import logging import tempfile import uuid import click import xmltodict from swh.deposit.client import PublicApiDepositClient -from swh.deposit.cli import cli +from swh.deposit.cli import deposit logger = logging.getLogger(__name__) class InputError(ValueError): """Input script error """ pass def generate_slug(): """Generate a slug (sample purposes). """ return str(uuid.uuid4()) def generate_metadata_file(name, external_id, authors): """Generate a temporary metadata file with the minimum required metadata This generates a xml file in a temporary location and returns the path to that file. This is up to the client of that function to clean up the temporary file. Args: name (str): Software's name external_id (str): External identifier (slug) or generated one authors (List[str]): List of author names Returns: Filepath to the metadata generated file """ _, tmpfile = tempfile.mkstemp(prefix='swh.deposit.cli.') # generate a metadata file with the minimum required metadata codemetadata = { 'entry': { '@xmlns': "http://www.w3.org/2005/Atom", '@xmlns:codemeta': "https://doi.org/10.5063/SCHEMA/CODEMETA-2.0", 'codemeta:name': name, 'codemeta:identifier': external_id, 'codemeta:author': [{ 'codemeta:name': author_name } for author_name in authors], }, } logging.debug('Temporary file: %s', tmpfile) logging.debug('Metadata dict to generate as xml: %s', codemetadata) s = xmltodict.unparse(codemetadata, pretty=True) logging.debug('Metadata dict as xml generated: %s', s) with open(tmpfile, 'w') as fp: fp.write(s) return tmpfile def _cleanup_tempfile(config): """Clean up the temporary metadata file generated. Args: config (Dict): A configuration dict with 2 important keys for that routine, 'cleanup_tempfile' (bool) and 'metadata' (path to eventually clean up) """ if config['cleanup_tempfile']: path = config['metadata'] if os.path.exists(path): os.unlink(path) def client_command_parse_input( username, password, archive, metadata, archive_deposit, metadata_deposit, collection, slug, partial, deposit_id, replace, url, status, name, authors): """Parse the client subcommand options and make sure the combination is acceptable*. If not, an InputError exception is raised explaining the issue. By acceptable, we mean: - A multipart deposit (create or update) needs both an existing software archive and an existing metadata file - A binary deposit (create/update) needs an existing software archive - A metadata deposit (create/update) needs an existing metadata file - A deposit update needs a deposit_id to be provided This won't prevent all failure cases though. The remaining errors are already dealt with the underlying api client. Raises: InputError explaining the issue Returns: dict with the following keys: 'archive': the software archive to deposit 'username': username 'password': associated password 'metadata': the metadata file to deposit 'collection': the username's associated client 'slug': the slug or external id identifying the deposit to make 'partial': if the deposit is partial or not 'client': instantiated class 'url': deposit's server main entry point 'deposit_type': deposit's type (binary, multipart, metadata) 'deposit_id': optional deposit identifier """ cleanup_tempfile = False try: if status and not deposit_id: raise InputError("Deposit id must be provided for status check") if status and deposit_id: # status is higher priority over deposit archive_deposit = False metadata_deposit = False archive = None metadata = None if archive_deposit and metadata_deposit: # too many flags use, remove redundant ones (-> multipart deposit) archive_deposit = False metadata_deposit = False if archive and not os.path.exists(archive): raise InputError('Software Archive %s must exist!' % archive) if not slug: # generate one as this is mandatory slug = generate_slug() if archive and not metadata: # we need to have the metadata if name and authors: metadata = generate_metadata_file(name, slug, authors) cleanup_tempfile = True else: raise InputError('Either metadata deposit file or (`--name` ' ' and `--author`) fields must be provided') if metadata_deposit: archive = None if archive_deposit: metadata = None if metadata_deposit and not metadata: raise InputError( "Metadata deposit filepath must be provided for metadata " "deposit") if metadata and not os.path.exists(metadata): raise InputError('Software Archive metadata %s must exist!' % ( metadata, )) if not status and not archive and not metadata: raise InputError( 'Please provide an actionable command. See --help for more ' 'information.') if replace and not deposit_id: raise InputError( 'To update an existing deposit, you must provide its id') client = PublicApiDepositClient({ 'url': url, 'auth': { 'username': username, 'password': password }, }) if not collection: # retrieve user's collection sd_content = client.service_document() if 'error' in sd_content: raise InputError('Service document retrieval: %s' % ( sd_content['error'], )) collection = sd_content[ 'service']['workspace']['collection']['sword:name'] return { 'archive': archive, 'username': username, 'password': password, 'metadata': metadata, 'cleanup_tempfile': cleanup_tempfile, 'collection': collection, 'slug': slug, 'in_progress': partial, 'client': client, 'url': url, 'deposit_id': deposit_id, 'replace': replace, } except Exception: # to be clean, cleanup prior to raise _cleanup_tempfile({ 'cleanup_tempfile': cleanup_tempfile, 'metadata': metadata }) raise def _subdict(d, keys): 'return a dict from d with only given keys' return {k: v for k, v in d.items() if k in keys} def deposit_status(config, logger): logger.debug('Status deposit') keys = ('collection', 'deposit_id') client = config['client'] return client.deposit_status( **_subdict(config, keys)) def deposit_create(config, logger): """Delegate the actual deposit to the deposit client. """ logger.debug('Create deposit') client = config['client'] keys = ('collection', 'archive', 'metadata', 'slug', 'in_progress') return client.deposit_create( **_subdict(config, keys)) def deposit_update(config, logger): """Delegate the actual deposit to the deposit client. """ logger.debug('Update deposit') client = config['client'] keys = ('collection', 'deposit_id', 'archive', 'metadata', 'slug', 'in_progress', 'replace') return client.deposit_update( **_subdict(config, keys)) -@cli.command() +@deposit.command() @click.option('--username', required=1, help="(Mandatory) User's name") @click.option('--password', required=1, help="(Mandatory) User's associated password") @click.option('--archive', help='(Optional) Software archive to deposit') @click.option('--metadata', help="(Optional) Path to xml metadata file. If not provided, this will use a file named .metadata.xml") # noqa @click.option('--archive-deposit/--no-archive-deposit', default=False, help='(Optional) Software archive only deposit') @click.option('--metadata-deposit/--no-metadata-deposit', default=False, help='(Optional) Metadata only deposit') @click.option('--collection', help="(Optional) User's collection. If not provided, this will be fetched.") # noqa @click.option('--slug', help="""(Optional) External system information identifier. If not provided, it will be generated""") # noqa @click.option('--partial/--no-partial', default=False, help='(Optional) The deposit will be partial, other deposits will have to take place to finalize it.') # noqa @click.option('--deposit-id', default=None, help='(Optional) Update an existing partial deposit with its identifier') # noqa @click.option('--replace/--no-replace', default=False, help='(Optional) Update by replacing existing metadata to a deposit') # noqa @click.option('--url', default='https://deposit.softwareheritage.org/1', help="(Optional) Deposit server api endpoint. By default, https://deposit.softwareheritage.org/1") # noqa @click.option('--status/--no-status', default=False, help="(Optional) Deposit's status") @click.option('--verbose/--no-verbose', default=False, help='Verbose mode') @click.option('--name', help='Software name') @click.option('--author', multiple=True, help='Software author(s), this can be repeated as many times' ' as there are authors') @click.pass_context def deposit(ctx, username, password, archive=None, metadata=None, archive_deposit=False, metadata_deposit=False, collection=None, slug=None, partial=False, deposit_id=None, replace=False, status=False, url='https://deposit.softwareheritage.org/1', verbose=False, name=None, author=None): """Software Heritage Public Deposit Client Create/Update deposit through the command line or access its status. More documentation can be found at https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html. """ config = {} try: logger.debug('Parsing cli options') config = client_command_parse_input( username, password, archive, metadata, archive_deposit, metadata_deposit, collection, slug, partial, deposit_id, replace, url, status, name, author) except InputError as e: msg = 'Problem during parsing options: %s' % e r = { 'error': msg, } logger.info(r) return 1 try: if verbose: logger.info("Parsed configuration: %s" % ( config, )) deposit_id = config['deposit_id'] if status and deposit_id: r = deposit_status(config, logger) elif not status and deposit_id: r = deposit_update(config, logger) elif not status and not deposit_id: r = deposit_create(config, logger) logger.info(r) finally: _cleanup_tempfile(config)