diff --git a/docs/sys-info.rst b/docs/sys-info.rst index 83a7676f..7a2200bc 100644 --- a/docs/sys-info.rst +++ b/docs/sys-info.rst @@ -1,60 +1,62 @@ Deployment of the swh-deposit ============================= As usual, the debian packaged is created and uploaded to the swh debian repository. Once the package is installed, we need to do a few things in regards to the database. Prepare the database setup (existence, connection, etc...). ----------------------------------------------------------- This is defined through the packaged ``swh.deposit.settings.production`` module and the expected **/etc/softwareheritage/deposit/server.yml**. As usual, the expected configuration files are deployed through our puppet manifest (cf. puppet-environment/swh-site, puppet-environment/swh-role, puppet-environment/swh-profile) Migrate/bootstrap the db schema ------------------------------- .. code:: shell sudo django-admin migrate --settings=swh.deposit.settings.production Load minimum defaults data -------------------------- .. code:: shell sudo django-admin loaddata \ --settings=swh.deposit.settings.production deposit_data This adds the minimal: - deposit request type 'archive' and 'metadata' - 'hal' collection Note: swh.deposit.fixtures.deposit\_data is packaged Add client and collection ------------------------- .. code:: shell - SWH_CONFIG_FILENAME=/etc/softwareheritage/deposit/server.yml \ - swh-deposit --platform production \ + swh-deposit \ + --config-file /etc/softwareheritage/deposit/server.yml \ + --platform production \ user create \ --collection \ --username \ --password This adds a user ```` which can access the collection ````. The password will be used for the authentication access to the deposit api. Note: - - If the collection does not exist, it is created alongside. + - If the collection does not exist, it is created alongside - The password is plain text but stored encrypted (so yes, for now we know the user's password) - - A production requirement for the cli to work is to set the - SWH_CONFIG_FILENAME environment variable + - For production platform, you must either set an + SWH_CONFIG_FILENAME environment variable or pass alongside the + `--config-file` parameter diff --git a/swh/deposit/cli.py b/swh/deposit/cli.py index 311d89f9..388affd1 100644 --- a/swh/deposit/cli.py +++ b/swh/deposit/cli.py @@ -1,158 +1,161 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import click from swh.deposit.config import setup_django_for CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) @click.group(context_settings=CONTEXT_SETTINGS) +@click.option('--config-file', '-C', default=None, + type=click.Path(exists=True, dir_okay=False,), + help="Optional extra configuration file.") @click.option('--platform', default='development', type=click.Choice(['development', 'production']), help='development or production platform') @click.pass_context -def cli(ctx, platform): - setup_django_for(platform) +def cli(ctx, config_file, platform): + setup_django_for(platform, config_file=config_file) @cli.group('user') @click.pass_context def user(ctx): """Manipulate user.""" pass def _create_collection(name): """Create the collection with name if it does not exist. Args: name (str): collection's name Returns: collection (DepositCollection): the existing collection object (created or not) """ # to avoid loading too early django namespaces from swh.deposit.models import DepositCollection try: collection = DepositCollection.objects.get(name=name) click.echo('Collection %s exists, nothing to do.' % name) except DepositCollection.DoesNotExist: click.echo('Create new collection %s' % name) collection = DepositCollection.objects.create(name=name) click.echo('Collection %s created' % name) return collection @user.command('create') @click.option('--username', required=True, help="User's name") @click.option('--password', required=True, help="Desired user's password (plain).") @click.option('--firstname', default='', help="User's first name") @click.option('--lastname', default='', help="User's last name") @click.option('--email', default='', help="User's email") @click.option('--collection', help="User's collection") @click.pass_context def user_create(ctx, username, password, firstname, lastname, email, collection): """Create a user with some needed information (password, collection) If the collection does not exist, the collection is then created alongside. The password is stored encrypted using django's utilies. """ # to avoid loading too early django namespaces from swh.deposit.models import DepositClient click.echo('collection: %s' % collection) # create the collection if it does not exist collection = _create_collection(collection) # user create/update try: user = DepositClient.objects.get(username=username) click.echo('User %s exists, updating information.' % user) user.set_password(password) except DepositClient.DoesNotExist: click.echo('Create new user %s' % username) user = DepositClient.objects.create_user( username=username, password=password) user.collections = [collection.id] user.first_name = firstname user.last_name = lastname user.email = email user.is_active = True user.save() click.echo('Information registered for user %s' % user) @user.command('list') @click.pass_context def user_list(ctx): """List existing users. This entrypoint is not paginated yet as there is not a lot of entry. """ # to avoid loading too early django namespaces from swh.deposit.models import DepositClient users = DepositClient.objects.all() if not users: output = 'Empty user list' else: output = '\n'.join((user.username for user in users)) click.echo(output) @cli.group('collection') @click.pass_context def collection(ctx): """Manipulate collection.""" pass @collection.command('create') @click.option('--name', required=True, help="Collection's name") @click.pass_context def collection_create(ctx, name): _create_collection(name) @collection.command('list') @click.pass_context def collection_list(ctx): """List existing collections. This entrypoint is not paginated yet as there is not a lot of entry. """ # to avoid loading too early django namespaces from swh.deposit.models import DepositCollection collections = DepositCollection.objects.all() if not collections: output = 'Empty collection list' else: output = '\n'.join((col.name for col in collections)) click.echo(output) def main(): return cli(auto_envvar_prefix='SWH_DEPOSIT') if __name__ == '__main__': main() diff --git a/swh/deposit/config.py b/swh/deposit/config.py index 1800d67c..40ba465f 100644 --- a/swh/deposit/config.py +++ b/swh/deposit/config.py @@ -1,101 +1,106 @@ # Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import logging from swh.core.config import SWHConfig from swh.scheduler import get_scheduler # IRIs (Internationalized Resource identifier) sword 2.0 specified EDIT_SE_IRI = 'edit_se_iri' EM_IRI = 'em_iri' CONT_FILE_IRI = 'cont_file_iri' SD_IRI = 'servicedocument' COL_IRI = 'upload' STATE_IRI = 'state_iri' PRIVATE_GET_RAW_CONTENT = 'private-download' PRIVATE_CHECK_DEPOSIT = 'check-deposit' PRIVATE_PUT_DEPOSIT = 'private-update' PRIVATE_GET_DEPOSIT_METADATA = 'private-read' PRIVATE_LIST_DEPOSITS = 'private-deposit-list' ARCHIVE_KEY = 'archive' METADATA_KEY = 'metadata' RAW_METADATA_KEY = 'raw-metadata' ARCHIVE_TYPE = 'archive' METADATA_TYPE = 'metadata' AUTHORIZED_PLATFORMS = ['development', 'production', 'testing'] DEPOSIT_STATUS_REJECTED = 'rejected' DEPOSIT_STATUS_PARTIAL = 'partial' DEPOSIT_STATUS_DEPOSITED = 'deposited' DEPOSIT_STATUS_VERIFIED = 'verified' DEPOSIT_STATUS_LOAD_SUCCESS = 'done' DEPOSIT_STATUS_LOAD_FAILURE = 'failed' # Revision author for deposit SWH_PERSON = { 'name': 'Software Heritage', 'fullname': 'Software Heritage', 'email': 'robot@softwareheritage.org' } -def setup_django_for(platform): +def setup_django_for(platform, config_file=None): """Setup function for command line tools (swh.deposit.create_user) to initialize the needed db access. Note: Do not import any django related module prior to this function call. Otherwise, this will raise an django.core.exceptions.ImproperlyConfigured error message. Args: platform (str): the platform the scheduling is running + config_file (str): Extra configuration file (typically for the + production platform) Raises: ValueError in case of wrong platform inputs. """ if platform not in AUTHORIZED_PLATFORMS: raise ValueError('Platform should be one of %s' % AUTHORIZED_PLATFORMS) os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swh.deposit.settings.%s' % platform) + if config_file: + os.environ.setdefault('SWH_CONFIG_FILENAME', config_file) + import django django.setup() class SWHDefaultConfig(SWHConfig): """Mixin intended to enrich views with SWH configuration. """ CONFIG_BASE_FILENAME = 'deposit/server' DEFAULT_CONFIG = { 'max_upload_size': ('int', 209715200), 'checks': ('bool', True), 'scheduler': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://localhost:5008/' } }) } ADDITIONAL_CONFIG = {} def __init__(self, **config): super().__init__() self.config = self.parse_config_file( additional_configs=[self.ADDITIONAL_CONFIG]) self.config.update(config) self.log = logging.getLogger('swh.deposit') if self.config['checks']: self.scheduler = get_scheduler(**self.config['scheduler'])