diff --git a/requirements-swh.txt b/requirements-swh.txt index 65ce072..3b4368c 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,3 +1,4 @@ swh.core >= 0.0.75 swh.model >= 0.0.18 swh.storage >= 0.0.153 +swh.scheduler diff --git a/setup.py b/setup.py index e88282c..5f24715 100755 --- a/setup.py +++ b/setup.py @@ -1,73 +1,75 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = 'requirements-%s.txt' % name else: reqf = 'requirements.txt' requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#'): continue requirements.append(line) return requirements setup( name='swh.loader.core', description='Software Heritage Base Loader', long_description=long_description, long_description_content_type='text/markdown', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DLDBASE', packages=find_packages(), # packages's modules scripts=[], # scripts to package install_requires=parse_requirements() + parse_requirements('swh'), setup_requires=['vcversioner'], extras_require={'testing': parse_requirements('test')}, vcversioner={}, include_package_data=True, entry_points=''' [swh.workers] loader.archive=swh.loader.package.archive:register loader.debian=swh.loader.package.debian:register loader.deposit=swh.loader.package.deposit:register loader.npm=swh.loader.package.npm:register loader.pypi=swh.loader.package.pypi:register + [swh.cli.subcommands] + loader=swh.loader.cli:run ''', classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', 'Funding': 'https://www.softwareheritage.org/donate', 'Source': 'https://forge.softwareheritage.org/source/swh-loader-core', }, ) diff --git a/swh/loader/cli.py b/swh/loader/cli.py new file mode 100644 index 0000000..06e35e5 --- /dev/null +++ b/swh/loader/cli.py @@ -0,0 +1,66 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import logging + +import click +import pkg_resources + +from swh.core.cli import CONTEXT_SETTINGS +from swh.scheduler.cli.utils import parse_options + + +logger = logging.getLogger(__name__) + + +LOADERS = {entry_point.name.split('.', 1)[1]: entry_point + for entry_point in pkg_resources.iter_entry_points('swh.workers') + if entry_point.name.split('.', 1)[0] == 'loader'} + + +SUPPORTED_LOADERS = list(LOADERS) + + +def get_loader(name: str, **kwargs): + """Given a loader name, instantiate it. + + Args: + name (str): Lister's name + conf (dict): Configuration dict (lister db cnx, policy, priority...) + + Returns: + An instantiated loader + + """ + if name not in LOADERS: + raise ValueError( + 'Invalid lister %s: only supported listers are %s' % + (name, SUPPORTED_LOADERS)) + + registry_entry = LOADERS[name].load()() + logger.debug(f'registry: {registry_entry}') + loader_cls = registry_entry['loader'] + logger.debug(f'loader class: {loader_cls}') + return loader_cls(**kwargs) + + +@click.command(name='run', context_settings=CONTEXT_SETTINGS) +@click.option('--type', '-t', help='Loader to run', + type=click.Choice(SUPPORTED_LOADERS)) +@click.option('--url', '-u', default=None, + help="Origin url to load") +@click.argument('options', nargs=-1) +@click.pass_context +def run(ctx, type, url, options): + """Loader cli tools + + Load an origin from its url with loader + + """ + (_, kw) = parse_options(options) + logger.debug(f'kw: {kw}') + loader = get_loader(type, url=url, **kw) + result = loader.load() + click.echo(result) diff --git a/swh/loader/package/archive/__init__.py b/swh/loader/package/archive/__init__.py index d39d5f5..bb824e7 100644 --- a/swh/loader/package/archive/__init__.py +++ b/swh/loader/package/archive/__init__.py @@ -1,14 +1,16 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Any, Mapping def register() -> Mapping[str, Any]: """Register the current worker module's definition""" + from .loader import ArchiveLoader return { 'task_modules': [f'{__name__}.tasks'], + 'loader': ArchiveLoader, } diff --git a/swh/loader/package/debian/__init__.py b/swh/loader/package/debian/__init__.py index d39d5f5..a28f3c9 100644 --- a/swh/loader/package/debian/__init__.py +++ b/swh/loader/package/debian/__init__.py @@ -1,14 +1,16 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Any, Mapping def register() -> Mapping[str, Any]: """Register the current worker module's definition""" + from .loader import DebianLoader return { 'task_modules': [f'{__name__}.tasks'], + 'loader': DebianLoader, } diff --git a/swh/loader/package/deposit/__init__.py b/swh/loader/package/deposit/__init__.py index d39d5f5..0de5433 100644 --- a/swh/loader/package/deposit/__init__.py +++ b/swh/loader/package/deposit/__init__.py @@ -1,14 +1,16 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Any, Mapping def register() -> Mapping[str, Any]: """Register the current worker module's definition""" + from .loader import DepositLoader return { 'task_modules': [f'{__name__}.tasks'], + 'loader': DepositLoader, } diff --git a/swh/loader/package/npm/__init__.py b/swh/loader/package/npm/__init__.py index d39d5f5..a35af88 100644 --- a/swh/loader/package/npm/__init__.py +++ b/swh/loader/package/npm/__init__.py @@ -1,14 +1,16 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Any, Mapping def register() -> Mapping[str, Any]: """Register the current worker module's definition""" + from .loader import NpmLoader return { 'task_modules': [f'{__name__}.tasks'], + 'loader': NpmLoader, } diff --git a/swh/loader/package/pypi/__init__.py b/swh/loader/package/pypi/__init__.py index d39d5f5..63f8fae 100644 --- a/swh/loader/package/pypi/__init__.py +++ b/swh/loader/package/pypi/__init__.py @@ -1,14 +1,16 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Any, Mapping def register() -> Mapping[str, Any]: """Register the current worker module's definition""" + from .loader import PyPILoader return { 'task_modules': [f'{__name__}.tasks'], + 'loader': PyPILoader, }