Page MenuHomeSoftware Heritage

D1236.id3931.diff
No OneTemporary

D1236.id3931.diff

diff --git a/docs/getting-started.rst b/docs/getting-started.rst
--- a/docs/getting-started.rst
+++ b/docs/getting-started.rst
@@ -115,24 +115,24 @@
.. code:: shell
- $ swh-deposit ---username name --password secret \
- --archive je-suis-gpl.tgz
+ $ swh-deposit deposit --username name --password secret \
+ --archive je-suis-gpl.tgz
with client's external identifier (``slug``)
.. code:: shell
- $ swh-deposit --username name --password secret \
- --archive je-suis-gpl.tgz \
- --slug je-suis-gpl
+ $ swh-deposit deposit --username name --password secret \
+ --archive je-suis-gpl.tgz \
+ --slug je-suis-gpl
to a specific client's collection
.. code:: shell
- $ swh-deposit --username name --password secret \
- --archive je-suis-gpl.tgz \
- --collection 'second-collection'
+ $ swh-deposit deposit --username name --password secret \
+ --archive je-suis-gpl.tgz \
+ --collection 'second-collection'
@@ -180,9 +180,9 @@
.. code:: shell
- $ swh-deposit --username name --password secret \
- --archive foo.tar.gz \
- --partial
+ $ swh-deposit deposit --username name --password secret \
+ --archive foo.tar.gz \
+ --partial
2. Add content or metadata to the deposit
@@ -193,34 +193,34 @@
.. code:: shell
- $ swh-deposit --username name --password secret \
- --archive add-foo.tar.gz \
- --deposit-id 42 \
- --partial
+ $ swh-deposit deposit --username name --password secret \
+ --archive add-foo.tar.gz \
+ --deposit-id 42 \
+ --partial
In case you want to add only one new archive without metadata:
.. code:: shell
- $ swh-deposit --username name --password secret \
- --archive add-foo.tar.gz \
- --archive-deposit \
- --deposit-id 42 \
- --partial \
+ $ swh-deposit deposit --username name --password secret \
+ --archive add-foo.tar.gz \
+ --archive-deposit \
+ --deposit-id 42 \
+ --partial \
If you want to add only metadata, use:
.. code:: shell
- $ swh-deposit --username name --password secret \
- --metadata add-foo.tar.gz.metadata.xml \
- --metadata-deposit \
- --deposit-id 42 \
- --partial
+ $ swh-deposit deposit --username name --password secret \
+ --metadata add-foo.tar.gz.metadata.xml \
+ --metadata-deposit \
+ --deposit-id 42 \
+ --partial
3. Finalize deposit
-~~~~~~~~~~~~~~~~~~~
+ ~~~~~~~~~~~~~~~~~~~
On your last addition, by not declaring it as ``--partial``, the
deposit will be considered as completed and its status will be changed
to ``deposited``.
@@ -243,10 +243,10 @@
.. code:: shell
- $ swh-deposit --username name --password secret \
- --deposit-id 11 \
- --archive updated-je-suis-gpl.tgz \
- --replace
+ $ swh-deposit deposit --username name --password secret \
+ --deposit-id 11 \
+ --archive updated-je-suis-gpl.tgz \
+ --replace
* update a loaded deposit with a new version:
@@ -255,9 +255,9 @@
.. code:: shell
- $ swh-deposit --username name --password secret \
- --archive je-suis-gpl-v2.tgz \
- --slug 'je-suis-gpl' \
+ $ swh-deposit deposit --username name --password secret \
+ --archive je-suis-gpl-v2.tgz \
+ --slug 'je-suis-gpl' \
@@ -268,7 +268,7 @@
.. code:: shell
- $ swh-deposit --username name --password secret --deposit-id '11' --status
+ $ swh-deposit deposit --username name --password secret --deposit-id '11' --status
.. code:: json
diff --git a/docs/sys-info.rst b/docs/sys-info.rst
--- a/docs/sys-info.rst
+++ b/docs/sys-info.rst
@@ -41,8 +41,9 @@
.. code:: shell
- SWH_CONFIG_FILENAME=/etc/softwareheritage/deposit/server.yml \
- swh-deposit --platform production \
+ swh-deposit \
+ --config-file /etc/softwareheritage/deposit/server.yml \
+ --platform production \
user create \
--collection <collection-name> \
--username <client-name> \
@@ -53,8 +54,9 @@
access to the deposit api.
Note:
- - If the collection does not exist, it is created alongside.
+ - If the collection does not exist, it is created alongside
- The password is plain text but stored encrypted (so yes, for now
we know the user's password)
- - A production requirement for the cli to work is to set the
- SWH_CONFIG_FILENAME environment variable
+ - For production platform, you must either set an
+ SWH_CONFIG_FILENAME environment variable or pass alongside the
+ `--config-file` parameter
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -53,7 +53,6 @@
entry_points='''
[console_scripts]
swh-deposit=swh.deposit.cli:main
- swh-deposit-client=swh.deposit.client.cli:main
''',
classifiers=[
"Programming Language :: Python :: 3",
diff --git a/swh/deposit/cli.py b/swh/deposit/cli.py
--- a/swh/deposit/cli.py
+++ b/swh/deposit/cli.py
@@ -4,20 +4,43 @@
# See top-level LICENSE file for more information
import click
+import os
+import logging
+import uuid
from swh.deposit.config import setup_django_for
+try:
+ from swh.deposit.client import PublicApiDepositClient
+except ImportError:
+ logging.warn("Optional client subcommand unavailable. "
+ "Install swh.deposit.client to be able to use it.")
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
@click.group(context_settings=CONTEXT_SETTINGS)
+@click.option('--config-file', '-C', default=None,
+ type=click.Path(exists=True, dir_okay=False,),
+ help="Optional extra configuration file.")
@click.option('--platform', default='development',
type=click.Choice(['development', 'production']),
help='development or production platform')
+@click.option('--verbose/--no-verbose', default=False,
+ help='Verbose mode')
@click.pass_context
-def cli(ctx, platform):
- setup_django_for(platform)
+def cli(ctx, config_file, platform, verbose):
+ logger = logging.getLogger(__name__)
+ logger.addHandler(logging.StreamHandler())
+ _loglevel = logging.DEBUG if verbose else logging.INFO
+ logger.setLevel(_loglevel)
+
+ ctx.ensure_object(dict)
+
+ # configuration happens here
+ setup_django_for(platform, config_file=config_file)
+
+ ctx.obj = {'loglevel': _loglevel}
@cli.group('user')
@@ -150,6 +173,278 @@
click.echo(output)
+class InputError(ValueError):
+ """Input script error
+
+ """
+ pass
+
+
+def generate_slug(prefix='swh-sample'):
+ """Generate a slug (sample purposes).
+
+ """
+ return '%s-%s' % (prefix, uuid.uuid4())
+
+
+def client_command_parse_input(
+ username, password, archive, metadata,
+ archive_deposit, metadata_deposit,
+ collection, slug, partial, deposit_id, replace,
+ url, status):
+ """Parse the client subcommand options and make sure the combination
+ is acceptable*. If not, an InputError exception is raised
+ explaining the issue.
+
+ By acceptable, we mean:
+
+ - A multipart deposit (create or update) needs both an
+ existing software archive and an existing metadata file
+
+ - A binary deposit (create/update) needs an existing
+ software archive
+
+ - A metadata deposit (create/update) needs an existing
+ metadata file
+
+ - A deposit update needs a deposit_id to be provided
+
+ This won't prevent all failure cases though. The remaining
+ errors are already dealt with the underlying api client.
+
+ Raises:
+ InputError explaining the issue
+
+ Returns:
+ dict with the following keys:
+
+ 'archive': the software archive to deposit
+ 'username': username
+ 'password': associated password
+ 'metadata': the metadata file to deposit
+ 'collection': the username's associated client
+ 'slug': the slug or external id identifying the deposit to make
+ 'partial': if the deposit is partial or not
+ 'client': instantiated class
+ 'url': deposit's server main entry point
+ 'deposit_type': deposit's type (binary, multipart, metadata)
+ 'deposit_id': optional deposit identifier
+
+ """
+ if status and not deposit_id:
+ raise InputError("Deposit id must be provided for status check")
+
+ if status and deposit_id: # status is higher priority over deposit
+ archive_deposit = False
+ metadata_deposit = False
+ archive = None
+ metadata = None
+
+ if archive_deposit and metadata_deposit:
+ # too many flags use, remove redundant ones (-> multipart deposit)
+ archive_deposit = False
+ metadata_deposit = False
+
+ if archive and not os.path.exists(archive):
+ raise InputError('Software Archive %s must exist!' % archive)
+
+ if archive and not metadata:
+ metadata = '%s.metadata.xml' % archive
+
+ if metadata_deposit:
+ archive = None
+
+ if archive_deposit:
+ metadata = None
+
+ if metadata_deposit and not metadata:
+ raise InputError(
+ "Metadata deposit filepath must be provided for metadata deposit")
+
+ if metadata and not os.path.exists(metadata):
+ raise InputError('Software Archive metadata %s must exist!' % metadata)
+
+ if not status and not archive and not metadata:
+ raise InputError(
+ 'Please provide an actionable command. See --help for more '
+ 'information.')
+
+ if replace and not deposit_id:
+ raise InputError(
+ 'To update an existing deposit, you must provide its id')
+
+ client = PublicApiDepositClient({
+ 'url': url,
+ 'auth': {
+ 'username': username,
+ 'password': password
+ },
+ })
+
+ if not collection:
+ # retrieve user's collection
+ sd_content = client.service_document()
+ if 'error' in sd_content:
+ raise InputError('Service document retrieval: %s' % (
+ sd_content['error'], ))
+ collection = sd_content['collection']
+
+ if not slug:
+ # generate slug
+ slug = generate_slug()
+
+ return {
+ 'archive': archive,
+ 'username': username,
+ 'password': password,
+ 'metadata': metadata,
+ 'collection': collection,
+ 'slug': slug,
+ 'partial': partial,
+ 'client': client,
+ 'url': url,
+ 'deposit_id': deposit_id,
+ 'replace': replace,
+ }
+
+
+def deposit_status(config, dry_run, logger):
+ logger.debug('Status deposit')
+ client = config['client']
+ collection = config['collection']
+ deposit_id = config['deposit_id']
+ if not dry_run:
+ r = client.deposit_status(collection, deposit_id, logger)
+ return r
+ return {}
+
+
+def deposit_create(config, dry_run, logger):
+ """Delegate the actual deposit to the deposit client.
+
+ """
+ logger.debug('Create deposit')
+
+ client = config['client']
+ collection = config['collection']
+ archive_path = config['archive']
+ metadata_path = config['metadata']
+ slug = config['slug']
+ in_progress = config['partial']
+ if not dry_run:
+ r = client.deposit_create(collection, slug, archive_path,
+ metadata_path, in_progress, logger)
+ return r
+ return {}
+
+
+def deposit_update(config, dry_run, logger):
+ """Delegate the actual deposit to the deposit client.
+
+ """
+ logger.debug('Update deposit')
+
+ client = config['client']
+ collection = config['collection']
+ deposit_id = config['deposit_id']
+ archive_path = config['archive']
+ metadata_path = config['metadata']
+ slug = config['slug']
+ in_progress = config['partial']
+ replace = config['replace']
+ if not dry_run:
+ r = client.deposit_update(collection, deposit_id, slug, archive_path,
+ metadata_path, in_progress, replace, logger)
+ return r
+ return {}
+
+
+@cli.command()
+@click.option('--username', required=1,
+ help="(Mandatory) User's name")
+@click.option('--password', required=1,
+ help="(Mandatory) User's associated password")
+@click.option('--archive',
+ help='(Optional) Software archive to deposit')
+@click.option('--metadata',
+ help="(Optional) Path to xml metadata file. If not provided, this will use a file named <archive>.metadata.xml") # noqa
+@click.option('--archive-deposit/--no-archive-deposit', default=False,
+ help='(Optional) Software archive only deposit')
+@click.option('--metadata-deposit/--no-metadata-deposit', default=False,
+ help='(Optional) Metadata only deposit')
+@click.option('--collection',
+ help="(Optional) User's collection. If not provided, this will be fetched.") # noqa
+@click.option('--slug',
+ help="""(Optional) External system information identifier. If not provided, it will be generated""") # noqa
+@click.option('--partial/--no-partial', default=False,
+ help='(Optional) The deposit will be partial, other deposits will have to take place to finalize it.') # noqa
+@click.option('--deposit-id', default=None,
+ help='(Optional) Update an existing partial deposit with its identifier') # noqa
+@click.option('--replace/--no-replace', default=False,
+ help='(Optional) Update by replacing existing metadata to a deposit') # noqa
+@click.option('--url', default='https://deposit.softwareheritage.org/1',
+ help="(Optional) Deposit server api endpoint. By default, https://deposit.softwareheritage.org/1") # noqa
+@click.option('--status/--no-status', default=False,
+ help="(Optional) Deposit's status")
+@click.option('--dry-run/--no-dry-run', default=False,
+ help='(Optional) No-op deposit')
+@click.option('--verbose/--no-verbose', default=False,
+ help='Verbose mode')
+@click.pass_context
+def deposit(ctx,
+ username, password, archive=None, metadata=None,
+ archive_deposit=False, metadata_deposit=False,
+ collection=None, slug=None, partial=False, deposit_id=None,
+ replace=False, status=False,
+ url='https://deposit.softwareheritage.org/1', dry_run=True,
+ verbose=False):
+ """Software Heritage Public Deposit Client
+
+ Create/Update deposit through the command line or access its
+ status.
+
+More documentation can be found at
+https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html.
+
+ """
+ logger = logging.getLogger(__name__)
+
+ if dry_run:
+ logger.info("**DRY RUN**")
+
+ config = {}
+
+ try:
+ logger.debug('Parsing cli options')
+ config = client_command_parse_input(
+ username, password, archive, metadata, archive_deposit,
+ metadata_deposit, collection, slug, partial, deposit_id,
+ replace, url, status)
+
+ except InputError as e:
+ msg = 'Problem during parsing options: %s' % e
+ r = {
+ 'error': msg,
+ }
+ logger.info(r)
+ return 1
+
+ if verbose:
+ logger.info("Parsed configuration: %s" % (
+ config, ))
+
+ deposit_id = config['deposit_id']
+
+ if status and deposit_id:
+ r = deposit_status(config, dry_run, logger)
+ elif not status and deposit_id:
+ r = deposit_update(config, dry_run, logger)
+ elif not status and not deposit_id:
+ r = deposit_create(config, dry_run, logger)
+
+ logger.info(r)
+
+
def main():
return cli(auto_envvar_prefix='SWH_DEPOSIT')
diff --git a/swh/deposit/client/cli.py b/swh/deposit/client/cli.py
deleted file mode 100755
--- a/swh/deposit/client/cli.py
+++ /dev/null
@@ -1,296 +0,0 @@
-# Copyright (C) 2018 The Software Heritage developers
-# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
-# See top-level LICENSE file for more information
-
-
-"""Script to demonstrate software deposit scenario to
-https://deposit.sofwareheritage.org.
-
-Use: python3 -m swh.deposit.client.cli --help
-
-Documentation: https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html # noqa
-
-"""
-
-import os
-import click
-import logging
-import uuid
-
-
-from . import PublicApiDepositClient
-
-
-class InputError(ValueError):
- """Input script error
-
- """
- pass
-
-
-def generate_slug(prefix='swh-sample'):
- """Generate a slug (sample purposes).
-
- """
- return '%s-%s' % (prefix, uuid.uuid4())
-
-
-def parse_cli_options(username, password, archive, metadata,
- archive_deposit, metadata_deposit,
- collection, slug, partial, deposit_id, replace,
- url, status):
- """Parse the cli options and make sure the combination is acceptable*.
- If not, an InputError exception is raised explaining the issue.
-
- By acceptable, we mean:
-
- - A multipart deposit (create or update) needs both an
- existing software archive and an existing metadata file
-
- - A binary deposit (create/update) needs an existing
- software archive
-
- - A metadata deposit (create/update) needs an existing
- metadata file
-
- - A deposit update needs a deposit_id to be provided
-
- This won't prevent all failure cases though. The remaining
- errors are already dealt with the underlying api client.
-
- Raises:
- InputError explaining the issue
-
- Returns:
- dict with the following keys:
-
- 'archive': the software archive to deposit
- 'username': username
- 'password': associated password
- 'metadata': the metadata file to deposit
- 'collection': the username's associated client
- 'slug': the slug or external id identifying the deposit to make
- 'partial': if the deposit is partial or not
- 'client': instantiated class
- 'url': deposit's server main entry point
- 'deposit_type': deposit's type (binary, multipart, metadata)
- 'deposit_id': optional deposit identifier
-
- """
- if status and not deposit_id:
- raise InputError("Deposit id must be provided for status check")
-
- if status and deposit_id: # status is higher priority over deposit
- archive_deposit = False
- metadata_deposit = False
- archive = None
- metadata = None
-
- if archive_deposit and metadata_deposit:
- # too many flags use, remove redundant ones (-> multipart deposit)
- archive_deposit = False
- metadata_deposit = False
-
- if archive and not os.path.exists(archive):
- raise InputError('Software Archive %s must exist!' % archive)
-
- if archive and not metadata:
- metadata = '%s.metadata.xml' % archive
-
- if metadata_deposit:
- archive = None
-
- if archive_deposit:
- metadata = None
-
- if metadata_deposit and not metadata:
- raise InputError(
- "Metadata deposit filepath must be provided for metadata deposit")
-
- if metadata and not os.path.exists(metadata):
- raise InputError('Software Archive metadata %s must exist!' % metadata)
-
- if not status and not archive and not metadata:
- raise InputError(
- 'Please provide an actionable command. See --help for more '
- 'information.')
-
- if replace and not deposit_id:
- raise InputError(
- 'To update an existing deposit, you must provide its id')
-
- client = PublicApiDepositClient({
- 'url': url,
- 'auth': {
- 'username': username,
- 'password': password
- },
- })
-
- if not collection:
- # retrieve user's collection
- sd_content = client.service_document()
- if 'error' in sd_content:
- raise InputError('Service document retrieval: %s' % (
- sd_content['error'], ))
- collection = sd_content['collection']
-
- if not slug:
- # generate slug
- slug = generate_slug()
-
- return {
- 'archive': archive,
- 'username': username,
- 'password': password,
- 'metadata': metadata,
- 'collection': collection,
- 'slug': slug,
- 'partial': partial,
- 'client': client,
- 'url': url,
- 'deposit_id': deposit_id,
- 'replace': replace,
- }
-
-
-def deposit_status(config, dry_run, log):
- log.debug('Status deposit')
- client = config['client']
- collection = config['collection']
- deposit_id = config['deposit_id']
- if not dry_run:
- r = client.deposit_status(collection, deposit_id, log)
- return r
- return {}
-
-
-def deposit_create(config, dry_run, log):
- """Delegate the actual deposit to the deposit client.
-
- """
- log.debug('Create deposit')
-
- client = config['client']
- collection = config['collection']
- archive_path = config['archive']
- metadata_path = config['metadata']
- slug = config['slug']
- in_progress = config['partial']
- if not dry_run:
- r = client.deposit_create(collection, slug, archive_path,
- metadata_path, in_progress, log)
- return r
- return {}
-
-
-def deposit_update(config, dry_run, log):
- """Delegate the actual deposit to the deposit client.
-
- """
- log.debug('Update deposit')
-
- client = config['client']
- collection = config['collection']
- deposit_id = config['deposit_id']
- archive_path = config['archive']
- metadata_path = config['metadata']
- slug = config['slug']
- in_progress = config['partial']
- replace = config['replace']
- if not dry_run:
- r = client.deposit_update(collection, deposit_id, slug, archive_path,
- metadata_path, in_progress, replace, log)
- return r
- return {}
-
-
-@click.command()
-@click.option('--username', required=1,
- help="(Mandatory) User's name")
-@click.option('--password', required=1,
- help="(Mandatory) User's associated password")
-@click.option('--archive',
- help='(Optional) Software archive to deposit')
-@click.option('--metadata',
- help="(Optional) Path to xml metadata file. If not provided, this will use a file named <archive>.metadata.xml") # noqa
-@click.option('--archive-deposit/--no-archive-deposit', default=False,
- help='(Optional) Software archive only deposit')
-@click.option('--metadata-deposit/--no-metadata-deposit', default=False,
- help='(Optional) Metadata only deposit')
-@click.option('--collection',
- help="(Optional) User's collection. If not provided, this will be fetched.") # noqa
-@click.option('--slug',
- help="""(Optional) External system information identifier. If not provided, it will be generated""") # noqa
-@click.option('--partial/--no-partial', default=False,
- help='(Optional) The deposit will be partial, other deposits will have to take place to finalize it.') # noqa
-@click.option('--deposit-id', default=None,
- help='(Optional) Update an existing partial deposit with its identifier') # noqa
-@click.option('--replace/--no-replace', default=False,
- help='(Optional) Update by replacing existing metadata to a deposit') # noqa
-@click.option('--url', default='https://deposit.softwareheritage.org/1',
- help="(Optional) Deposit server api endpoint. By default, https://deposit.softwareheritage.org/1") # noqa
-@click.option('--status/--no-status', default=False,
- help="(Optional) Deposit's status")
-@click.option('--dry-run/--no-dry-run', default=False,
- help='(Optional) No-op deposit')
-@click.option('--verbose/--no-verbose', default=False,
- help='Verbose mode')
-def main(username, password, archive=None, metadata=None,
- archive_deposit=False, metadata_deposit=False,
- collection=None, slug=None, partial=False, deposit_id=None,
- replace=False, status=False,
- url='https://deposit.softwareheritage.org/1', dry_run=True,
- verbose=False):
- """Software Heritage Deposit client - Create (or update partial)
-deposit through the command line.
-
-More documentation can be found at
-https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html.
-
- """
-
- log = logging.getLogger('swh-deposit')
- log.addHandler(logging.StreamHandler())
- _loglevel = logging.DEBUG if verbose else logging.INFO
- log.setLevel(_loglevel)
-
- if dry_run:
- log.info("**DRY RUN**")
-
- config = {}
-
- try:
- log.debug('Parsing cli options')
- config = parse_cli_options(
- username, password, archive, metadata, archive_deposit,
- metadata_deposit, collection, slug, partial, deposit_id,
- replace, url, status)
-
- except InputError as e:
- msg = 'Problem during parsing options: %s' % e
- r = {
- 'error': msg,
- }
- log.info(r)
- return 1
-
- if verbose:
- log.info("Parsed configuration: %s" % (
- config, ))
-
- deposit_id = config['deposit_id']
-
- if status and deposit_id:
- r = deposit_status(config, dry_run, log)
- elif not status and deposit_id:
- r = deposit_update(config, dry_run, log)
- elif not status and not deposit_id:
- r = deposit_create(config, dry_run, log)
-
- log.info(r)
-
-
-if __name__ == '__main__':
- main()
diff --git a/swh/deposit/config.py b/swh/deposit/config.py
--- a/swh/deposit/config.py
+++ b/swh/deposit/config.py
@@ -46,7 +46,7 @@
}
-def setup_django_for(platform):
+def setup_django_for(platform, config_file=None):
"""Setup function for command line tools (swh.deposit.create_user) to
initialize the needed db access.
@@ -57,6 +57,8 @@
Args:
platform (str): the platform the scheduling is running
+ config_file (str): Extra configuration file (typically for the
+ production platform)
Raises:
ValueError in case of wrong platform inputs.
@@ -68,6 +70,9 @@
os.environ.setdefault('DJANGO_SETTINGS_MODULE',
'swh.deposit.settings.%s' % platform)
+ if config_file:
+ os.environ.setdefault('SWH_CONFIG_FILENAME', config_file)
+
import django
django.setup()

File Metadata

Mime Type
text/plain
Expires
Jul 3 2025, 6:12 PM (4 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3218825

Event Timeline