diff --git a/docs/getting-started.rst b/docs/getting-started.rst --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -115,24 +115,24 @@ .. code:: shell - $ swh-deposit ---username name --password secret \ - --archive je-suis-gpl.tgz + $ swh-deposit client --username name --password secret \ + --archive je-suis-gpl.tgz with client's external identifier (``slug``) .. code:: shell - $ swh-deposit --username name --password secret \ - --archive je-suis-gpl.tgz \ - --slug je-suis-gpl + $ swh-deposit client --username name --password secret \ + --archive je-suis-gpl.tgz \ + --slug je-suis-gpl to a specific client's collection .. code:: shell - $ swh-deposit --username name --password secret \ - --archive je-suis-gpl.tgz \ - --collection 'second-collection' + $ swh-deposit client --username name --password secret \ + --archive je-suis-gpl.tgz \ + --collection 'second-collection' @@ -180,9 +180,9 @@ .. code:: shell - $ swh-deposit --username name --password secret \ - --archive foo.tar.gz \ - --partial + $ swh-deposit client --username name --password secret \ + --archive foo.tar.gz \ + --partial 2. Add content or metadata to the deposit @@ -193,31 +193,31 @@ .. code:: shell - $ swh-deposit --username name --password secret \ - --archive add-foo.tar.gz \ - --deposit-id 42 \ - --partial + $ swh-deposit client --username name --password secret \ + --archive add-foo.tar.gz \ + --deposit-id 42 \ + --partial In case you want to add only one new archive without metadata: .. code:: shell - $ swh-deposit --username name --password secret \ - --archive add-foo.tar.gz \ - --archive-deposit \ - --deposit-id 42 \ - --partial \ + $ swh-deposit client --username name --password secret \ + --archive add-foo.tar.gz \ + --archive-deposit \ + --deposit-id 42 \ + --partial \ If you want to add only metadata, use: .. code:: shell - $ swh-deposit --username name --password secret \ - --metadata add-foo.tar.gz.metadata.xml \ - --metadata-deposit \ - --deposit-id 42 \ - --partial + $ swh-deposit client --username name --password secret \ + --metadata add-foo.tar.gz.metadata.xml \ + --metadata-deposit \ + --deposit-id 42 \ + --partial 3. Finalize deposit ~~~~~~~~~~~~~~~~~~~ @@ -243,10 +243,10 @@ .. code:: shell - $ swh-deposit --username name --password secret \ - --deposit-id 11 \ - --archive updated-je-suis-gpl.tgz \ - --replace + $ swh-deposit client --username name --password secret \ + --deposit-id 11 \ + --archive updated-je-suis-gpl.tgz \ + --replace * update a loaded deposit with a new version: @@ -255,9 +255,9 @@ .. code:: shell - $ swh-deposit --username name --password secret \ - --archive je-suis-gpl-v2.tgz \ - --slug 'je-suis-gpl' \ + $ swh-deposit client --username name --password secret \ + --archive je-suis-gpl-v2.tgz \ + --slug 'je-suis-gpl' \ @@ -268,7 +268,7 @@ .. code:: shell - $ swh-deposit --username name --password secret --deposit-id '11' --status + $ swh-deposit client --username name --password secret --deposit-id '11' --status .. code:: json diff --git a/docs/sys-info.rst b/docs/sys-info.rst --- a/docs/sys-info.rst +++ b/docs/sys-info.rst @@ -41,8 +41,9 @@ .. code:: shell - SWH_CONFIG_FILENAME=/etc/softwareheritage/deposit/server.yml \ - swh-deposit --platform production \ + swh-deposit \ + --config-file /etc/softwareheritage/deposit/server.yml \ + --platform production \ user create \ --collection \ --username \ @@ -53,8 +54,9 @@ access to the deposit api. Note: - - If the collection does not exist, it is created alongside. + - If the collection does not exist, it is created alongside - The password is plain text but stored encrypted (so yes, for now we know the user's password) - - A production requirement for the cli to work is to set the - SWH_CONFIG_FILENAME environment variable + - For production platform, you must either set an + SWH_CONFIG_FILENAME environment variable or pass alongside the + `--config-file` parameter diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -53,7 +53,6 @@ entry_points=''' [console_scripts] swh-deposit=swh.deposit.cli:main - swh-deposit-client=swh.deposit.client.cli:main ''', classifiers=[ "Programming Language :: Python :: 3", diff --git a/swh/deposit/cli.py b/swh/deposit/cli.py --- a/swh/deposit/cli.py +++ b/swh/deposit/cli.py @@ -4,20 +4,43 @@ # See top-level LICENSE file for more information import click +import os +import logging +import uuid from swh.deposit.config import setup_django_for +try: + from swh.deposit.client import PublicApiDepositClient +except ImportError: + logging.warn("Optional client subcommand unavailable. " + "Install swh.deposit.client to be able to use it.") CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) @click.group(context_settings=CONTEXT_SETTINGS) +@click.option('--config-file', '-C', default=None, + type=click.Path(exists=True, dir_okay=False,), + help="Optional extra configuration file.") @click.option('--platform', default='development', type=click.Choice(['development', 'production']), help='development or production platform') +@click.option('--verbose/--no-verbose', default=False, + help='Verbose mode') @click.pass_context -def cli(ctx, platform): - setup_django_for(platform) +def cli(ctx, config_file, platform, verbose): + logger = logging.getLogger(__name__) + logger.addHandler(logging.StreamHandler()) + _loglevel = logging.DEBUG if verbose else logging.INFO + logger.setLevel(_loglevel) + + ctx.ensure_object(dict) + + # configuration happens here + setup_django_for(platform, config_file=config_file) + + ctx.obj = {'loglevel': _loglevel} @cli.group('user') @@ -150,6 +173,278 @@ click.echo(output) +class InputError(ValueError): + """Input script error + + """ + pass + + +def generate_slug(prefix='swh-sample'): + """Generate a slug (sample purposes). + + """ + return '%s-%s' % (prefix, uuid.uuid4()) + + +def client_command_parse_input( + username, password, archive, metadata, + archive_deposit, metadata_deposit, + collection, slug, partial, deposit_id, replace, + url, status): + """Parse the client subcommand options and make sure the combination + is acceptable*. If not, an InputError exception is raised + explaining the issue. + + By acceptable, we mean: + + - A multipart deposit (create or update) needs both an + existing software archive and an existing metadata file + + - A binary deposit (create/update) needs an existing + software archive + + - A metadata deposit (create/update) needs an existing + metadata file + + - A deposit update needs a deposit_id to be provided + + This won't prevent all failure cases though. The remaining + errors are already dealt with the underlying api client. + + Raises: + InputError explaining the issue + + Returns: + dict with the following keys: + + 'archive': the software archive to deposit + 'username': username + 'password': associated password + 'metadata': the metadata file to deposit + 'collection': the username's associated client + 'slug': the slug or external id identifying the deposit to make + 'partial': if the deposit is partial or not + 'client': instantiated class + 'url': deposit's server main entry point + 'deposit_type': deposit's type (binary, multipart, metadata) + 'deposit_id': optional deposit identifier + + """ + if status and not deposit_id: + raise InputError("Deposit id must be provided for status check") + + if status and deposit_id: # status is higher priority over deposit + archive_deposit = False + metadata_deposit = False + archive = None + metadata = None + + if archive_deposit and metadata_deposit: + # too many flags use, remove redundant ones (-> multipart deposit) + archive_deposit = False + metadata_deposit = False + + if archive and not os.path.exists(archive): + raise InputError('Software Archive %s must exist!' % archive) + + if archive and not metadata: + metadata = '%s.metadata.xml' % archive + + if metadata_deposit: + archive = None + + if archive_deposit: + metadata = None + + if metadata_deposit and not metadata: + raise InputError( + "Metadata deposit filepath must be provided for metadata deposit") + + if metadata and not os.path.exists(metadata): + raise InputError('Software Archive metadata %s must exist!' % metadata) + + if not status and not archive and not metadata: + raise InputError( + 'Please provide an actionable command. See --help for more ' + 'information.') + + if replace and not deposit_id: + raise InputError( + 'To update an existing deposit, you must provide its id') + + client = PublicApiDepositClient({ + 'url': url, + 'auth': { + 'username': username, + 'password': password + }, + }) + + if not collection: + # retrieve user's collection + sd_content = client.service_document() + if 'error' in sd_content: + raise InputError('Service document retrieval: %s' % ( + sd_content['error'], )) + collection = sd_content['collection'] + + if not slug: + # generate slug + slug = generate_slug() + + return { + 'archive': archive, + 'username': username, + 'password': password, + 'metadata': metadata, + 'collection': collection, + 'slug': slug, + 'partial': partial, + 'client': client, + 'url': url, + 'deposit_id': deposit_id, + 'replace': replace, + } + + +def deposit_status(config, dry_run, logger): + logger.debug('Status deposit') + client = config['client'] + collection = config['collection'] + deposit_id = config['deposit_id'] + if not dry_run: + r = client.deposit_status(collection, deposit_id, logger) + return r + return {} + + +def deposit_create(config, dry_run, logger): + """Delegate the actual deposit to the deposit client. + + """ + logger.debug('Create deposit') + + client = config['client'] + collection = config['collection'] + archive_path = config['archive'] + metadata_path = config['metadata'] + slug = config['slug'] + in_progress = config['partial'] + if not dry_run: + r = client.deposit_create(collection, slug, archive_path, + metadata_path, in_progress, logger) + return r + return {} + + +def deposit_update(config, dry_run, logger): + """Delegate the actual deposit to the deposit client. + + """ + logger.debug('Update deposit') + + client = config['client'] + collection = config['collection'] + deposit_id = config['deposit_id'] + archive_path = config['archive'] + metadata_path = config['metadata'] + slug = config['slug'] + in_progress = config['partial'] + replace = config['replace'] + if not dry_run: + r = client.deposit_update(collection, deposit_id, slug, archive_path, + metadata_path, in_progress, replace, logger) + return r + return {} + + +@cli.command() +@click.option('--username', required=1, + help="(Mandatory) User's name") +@click.option('--password', required=1, + help="(Mandatory) User's associated password") +@click.option('--archive', + help='(Optional) Software archive to deposit') +@click.option('--metadata', + help="(Optional) Path to xml metadata file. If not provided, this will use a file named .metadata.xml") # noqa +@click.option('--archive-deposit/--no-archive-deposit', default=False, + help='(Optional) Software archive only deposit') +@click.option('--metadata-deposit/--no-metadata-deposit', default=False, + help='(Optional) Metadata only deposit') +@click.option('--collection', + help="(Optional) User's collection. If not provided, this will be fetched.") # noqa +@click.option('--slug', + help="""(Optional) External system information identifier. If not provided, it will be generated""") # noqa +@click.option('--partial/--no-partial', default=False, + help='(Optional) The deposit will be partial, other deposits will have to take place to finalize it.') # noqa +@click.option('--deposit-id', default=None, + help='(Optional) Update an existing partial deposit with its identifier') # noqa +@click.option('--replace/--no-replace', default=False, + help='(Optional) Update by replacing existing metadata to a deposit') # noqa +@click.option('--url', default='https://deposit.softwareheritage.org/1', + help="(Optional) Deposit server api endpoint. By default, https://deposit.softwareheritage.org/1") # noqa +@click.option('--status/--no-status', default=False, + help="(Optional) Deposit's status") +@click.option('--dry-run/--no-dry-run', default=False, + help='(Optional) No-op deposit') +@click.option('--verbose/--no-verbose', default=False, + help='Verbose mode') +@click.pass_context +def client(ctx, + username, password, archive=None, metadata=None, + archive_deposit=False, metadata_deposit=False, + collection=None, slug=None, partial=False, deposit_id=None, + replace=False, status=False, + url='https://deposit.softwareheritage.org/1', dry_run=True, + verbose=False): + """Software Heritage Public Deposit Client + + Create/Update deposit through the command line or access its + status. + +More documentation can be found at +https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html. + + """ + logger = logging.getLogger(__name__) + + if dry_run: + logger.info("**DRY RUN**") + + config = {} + + try: + logger.debug('Parsing cli options') + config = client_command_parse_input( + username, password, archive, metadata, archive_deposit, + metadata_deposit, collection, slug, partial, deposit_id, + replace, url, status) + + except InputError as e: + msg = 'Problem during parsing options: %s' % e + r = { + 'error': msg, + } + logger.info(r) + return 1 + + if verbose: + logger.info("Parsed configuration: %s" % ( + config, )) + + deposit_id = config['deposit_id'] + + if status and deposit_id: + r = deposit_status(config, dry_run, logger) + elif not status and deposit_id: + r = deposit_update(config, dry_run, logger) + elif not status and not deposit_id: + r = deposit_create(config, dry_run, logger) + + logger.info(r) + + def main(): return cli(auto_envvar_prefix='SWH_DEPOSIT') diff --git a/swh/deposit/client/cli.py b/swh/deposit/client/cli.py deleted file mode 100755 --- a/swh/deposit/client/cli.py +++ /dev/null @@ -1,296 +0,0 @@ -# Copyright (C) 2018 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - - -"""Script to demonstrate software deposit scenario to -https://deposit.sofwareheritage.org. - -Use: python3 -m swh.deposit.client.cli --help - -Documentation: https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html # noqa - -""" - -import os -import click -import logging -import uuid - - -from . import PublicApiDepositClient - - -class InputError(ValueError): - """Input script error - - """ - pass - - -def generate_slug(prefix='swh-sample'): - """Generate a slug (sample purposes). - - """ - return '%s-%s' % (prefix, uuid.uuid4()) - - -def parse_cli_options(username, password, archive, metadata, - archive_deposit, metadata_deposit, - collection, slug, partial, deposit_id, replace, - url, status): - """Parse the cli options and make sure the combination is acceptable*. - If not, an InputError exception is raised explaining the issue. - - By acceptable, we mean: - - - A multipart deposit (create or update) needs both an - existing software archive and an existing metadata file - - - A binary deposit (create/update) needs an existing - software archive - - - A metadata deposit (create/update) needs an existing - metadata file - - - A deposit update needs a deposit_id to be provided - - This won't prevent all failure cases though. The remaining - errors are already dealt with the underlying api client. - - Raises: - InputError explaining the issue - - Returns: - dict with the following keys: - - 'archive': the software archive to deposit - 'username': username - 'password': associated password - 'metadata': the metadata file to deposit - 'collection': the username's associated client - 'slug': the slug or external id identifying the deposit to make - 'partial': if the deposit is partial or not - 'client': instantiated class - 'url': deposit's server main entry point - 'deposit_type': deposit's type (binary, multipart, metadata) - 'deposit_id': optional deposit identifier - - """ - if status and not deposit_id: - raise InputError("Deposit id must be provided for status check") - - if status and deposit_id: # status is higher priority over deposit - archive_deposit = False - metadata_deposit = False - archive = None - metadata = None - - if archive_deposit and metadata_deposit: - # too many flags use, remove redundant ones (-> multipart deposit) - archive_deposit = False - metadata_deposit = False - - if archive and not os.path.exists(archive): - raise InputError('Software Archive %s must exist!' % archive) - - if archive and not metadata: - metadata = '%s.metadata.xml' % archive - - if metadata_deposit: - archive = None - - if archive_deposit: - metadata = None - - if metadata_deposit and not metadata: - raise InputError( - "Metadata deposit filepath must be provided for metadata deposit") - - if metadata and not os.path.exists(metadata): - raise InputError('Software Archive metadata %s must exist!' % metadata) - - if not status and not archive and not metadata: - raise InputError( - 'Please provide an actionable command. See --help for more ' - 'information.') - - if replace and not deposit_id: - raise InputError( - 'To update an existing deposit, you must provide its id') - - client = PublicApiDepositClient({ - 'url': url, - 'auth': { - 'username': username, - 'password': password - }, - }) - - if not collection: - # retrieve user's collection - sd_content = client.service_document() - if 'error' in sd_content: - raise InputError('Service document retrieval: %s' % ( - sd_content['error'], )) - collection = sd_content['collection'] - - if not slug: - # generate slug - slug = generate_slug() - - return { - 'archive': archive, - 'username': username, - 'password': password, - 'metadata': metadata, - 'collection': collection, - 'slug': slug, - 'partial': partial, - 'client': client, - 'url': url, - 'deposit_id': deposit_id, - 'replace': replace, - } - - -def deposit_status(config, dry_run, log): - log.debug('Status deposit') - client = config['client'] - collection = config['collection'] - deposit_id = config['deposit_id'] - if not dry_run: - r = client.deposit_status(collection, deposit_id, log) - return r - return {} - - -def deposit_create(config, dry_run, log): - """Delegate the actual deposit to the deposit client. - - """ - log.debug('Create deposit') - - client = config['client'] - collection = config['collection'] - archive_path = config['archive'] - metadata_path = config['metadata'] - slug = config['slug'] - in_progress = config['partial'] - if not dry_run: - r = client.deposit_create(collection, slug, archive_path, - metadata_path, in_progress, log) - return r - return {} - - -def deposit_update(config, dry_run, log): - """Delegate the actual deposit to the deposit client. - - """ - log.debug('Update deposit') - - client = config['client'] - collection = config['collection'] - deposit_id = config['deposit_id'] - archive_path = config['archive'] - metadata_path = config['metadata'] - slug = config['slug'] - in_progress = config['partial'] - replace = config['replace'] - if not dry_run: - r = client.deposit_update(collection, deposit_id, slug, archive_path, - metadata_path, in_progress, replace, log) - return r - return {} - - -@click.command() -@click.option('--username', required=1, - help="(Mandatory) User's name") -@click.option('--password', required=1, - help="(Mandatory) User's associated password") -@click.option('--archive', - help='(Optional) Software archive to deposit') -@click.option('--metadata', - help="(Optional) Path to xml metadata file. If not provided, this will use a file named .metadata.xml") # noqa -@click.option('--archive-deposit/--no-archive-deposit', default=False, - help='(Optional) Software archive only deposit') -@click.option('--metadata-deposit/--no-metadata-deposit', default=False, - help='(Optional) Metadata only deposit') -@click.option('--collection', - help="(Optional) User's collection. If not provided, this will be fetched.") # noqa -@click.option('--slug', - help="""(Optional) External system information identifier. If not provided, it will be generated""") # noqa -@click.option('--partial/--no-partial', default=False, - help='(Optional) The deposit will be partial, other deposits will have to take place to finalize it.') # noqa -@click.option('--deposit-id', default=None, - help='(Optional) Update an existing partial deposit with its identifier') # noqa -@click.option('--replace/--no-replace', default=False, - help='(Optional) Update by replacing existing metadata to a deposit') # noqa -@click.option('--url', default='https://deposit.softwareheritage.org/1', - help="(Optional) Deposit server api endpoint. By default, https://deposit.softwareheritage.org/1") # noqa -@click.option('--status/--no-status', default=False, - help="(Optional) Deposit's status") -@click.option('--dry-run/--no-dry-run', default=False, - help='(Optional) No-op deposit') -@click.option('--verbose/--no-verbose', default=False, - help='Verbose mode') -def main(username, password, archive=None, metadata=None, - archive_deposit=False, metadata_deposit=False, - collection=None, slug=None, partial=False, deposit_id=None, - replace=False, status=False, - url='https://deposit.softwareheritage.org/1', dry_run=True, - verbose=False): - """Software Heritage Deposit client - Create (or update partial) -deposit through the command line. - -More documentation can be found at -https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html. - - """ - - log = logging.getLogger('swh-deposit') - log.addHandler(logging.StreamHandler()) - _loglevel = logging.DEBUG if verbose else logging.INFO - log.setLevel(_loglevel) - - if dry_run: - log.info("**DRY RUN**") - - config = {} - - try: - log.debug('Parsing cli options') - config = parse_cli_options( - username, password, archive, metadata, archive_deposit, - metadata_deposit, collection, slug, partial, deposit_id, - replace, url, status) - - except InputError as e: - msg = 'Problem during parsing options: %s' % e - r = { - 'error': msg, - } - log.info(r) - return 1 - - if verbose: - log.info("Parsed configuration: %s" % ( - config, )) - - deposit_id = config['deposit_id'] - - if status and deposit_id: - r = deposit_status(config, dry_run, log) - elif not status and deposit_id: - r = deposit_update(config, dry_run, log) - elif not status and not deposit_id: - r = deposit_create(config, dry_run, log) - - log.info(r) - - -if __name__ == '__main__': - main() diff --git a/swh/deposit/config.py b/swh/deposit/config.py --- a/swh/deposit/config.py +++ b/swh/deposit/config.py @@ -46,7 +46,7 @@ } -def setup_django_for(platform): +def setup_django_for(platform, config_file=None): """Setup function for command line tools (swh.deposit.create_user) to initialize the needed db access. @@ -57,6 +57,8 @@ Args: platform (str): the platform the scheduling is running + config_file (str): Extra configuration file (typically for the + production platform) Raises: ValueError in case of wrong platform inputs. @@ -68,6 +70,9 @@ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swh.deposit.settings.%s' % platform) + if config_file: + os.environ.setdefault('SWH_CONFIG_FILENAME', config_file) + import django django.setup()