diff --git a/requirements-swh.txt b/requirements-swh.txt index 6410dfe..bbdb106 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,2 +1,2 @@ -swh.core >= 0.0.77 +swh.core >= 0.3 swh.scheduler >= 0.0.58 diff --git a/setup.py b/setup.py index bac190c..620af95 100755 --- a/setup.py +++ b/setup.py @@ -1,85 +1,85 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from io import open from os import path from setuptools import find_packages, setup here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = "requirements-%s.txt" % name else: reqf = "requirements.txt" requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith("#"): continue requirements.append(line) return requirements setup( name="swh.lister", description="Software Heritage lister", long_description=long_description, long_description_content_type="text/markdown", python_requires=">=3.7", author="Software Heritage developers", author_email="swh-devel@inria.fr", url="https://forge.softwareheritage.org/diffusion/DLSGH/", packages=find_packages(), install_requires=parse_requirements() + parse_requirements("swh"), tests_require=parse_requirements("test"), setup_requires=["setuptools-scm"], extras_require={"testing": parse_requirements("test")}, use_scm_version=True, include_package_data=True, entry_points=""" [swh.cli.subcommands] - lister=swh.lister.cli:lister + lister=swh.lister.cli [swh.workers] lister.bitbucket=swh.lister.bitbucket:register lister.cgit=swh.lister.cgit:register lister.cran=swh.lister.cran:register lister.debian=swh.lister.debian:register lister.gitea=swh.lister.gitea:register lister.github=swh.lister.github:register lister.gitlab=swh.lister.gitlab:register lister.gnu=swh.lister.gnu:register lister.launchpad=swh.lister.launchpad:register lister.npm=swh.lister.npm:register lister.packagist=swh.lister.packagist:register lister.phabricator=swh.lister.phabricator:register lister.pypi=swh.lister.pypi:register """, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ "Bug Reports": "https://forge.softwareheritage.org/maniphest", "Funding": "https://www.softwareheritage.org/donate", "Source": "https://forge.softwareheritage.org/source/swh-lister", "Documentation": "https://docs.softwareheritage.org/devel/swh-lister/", }, ) diff --git a/swh/lister/cli.py b/swh/lister/cli.py index 523b273..d90ddbf 100644 --- a/swh/lister/cli.py +++ b/swh/lister/cli.py @@ -1,148 +1,148 @@ # Copyright (C) 2018-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from copy import deepcopy import logging # WARNING: do not import unnecessary things here to keep cli startup time under # control import os import click -from swh.core.cli import CONTEXT_SETTINGS +from swh.core.cli import CONTEXT_SETTINGS, swh as swh_cli_group from swh.lister import LISTERS, SUPPORTED_LISTERS, get_lister logger = logging.getLogger(__name__) # the key in this dict is the suffix used to match new task-type to be added. # For example for a task which function name is "list_gitlab_full', the default # value used when inserting a new task-type in the scheduler db will be the one # under the 'full' key below (because it matches xxx_full). DEFAULT_TASK_TYPE = { "full": { # for tasks like 'list_xxx_full()' "default_interval": "90 days", "min_interval": "90 days", "max_interval": "90 days", "backoff_factor": 1, }, "*": { # value if not suffix matches "default_interval": "1 day", "min_interval": "1 day", "max_interval": "1 day", "backoff_factor": 1, }, } -@click.group(name="lister", context_settings=CONTEXT_SETTINGS) +@swh_cli_group.group(name="lister", context_settings=CONTEXT_SETTINGS) @click.option( "--config-file", "-C", default=None, type=click.Path(exists=True, dir_okay=False,), help="Configuration file.", ) @click.option( "--db-url", "-d", default=None, help="SQLAlchemy DB URL; see " "", ) # noqa @click.pass_context def lister(ctx, config_file, db_url): """Software Heritage Lister tools.""" from swh.core import config ctx.ensure_object(dict) if not config_file: config_file = os.environ.get("SWH_CONFIG_FILENAME") conf = config.read(config_file) if db_url: conf["lister"] = {"cls": "local", "args": {"db": db_url}} ctx.obj["config"] = conf @lister.command(name="db-init", context_settings=CONTEXT_SETTINGS) @click.option( "--drop-tables", "-D", is_flag=True, default=False, help="Drop tables before creating the database schema", ) @click.pass_context def db_init(ctx, drop_tables): """Initialize the database model for given listers. """ from sqlalchemy import create_engine from swh.lister.core.models import initialize cfg = ctx.obj["config"] lister_cfg = cfg["lister"] if lister_cfg["cls"] != "local": click.echo("A local lister configuration is required") ctx.exit(1) db_url = lister_cfg["args"]["db"] db_engine = create_engine(db_url) registry = {} for lister, entrypoint in LISTERS.items(): logger.info("Loading lister %s", lister) registry[lister] = entrypoint.load()() logger.info("Initializing database") initialize(db_engine, drop_tables) for lister, entrypoint in LISTERS.items(): registry_entry = registry[lister] init_hook = registry_entry.get("init") if callable(init_hook): logger.info("Calling init hook for %s", lister) init_hook(db_engine) @lister.command( name="run", context_settings=CONTEXT_SETTINGS, help="Trigger a full listing run for a particular forge " "instance. The output of this listing results in " '"oneshot" tasks in the scheduler db with a priority ' "defined by the user", ) @click.option( "--lister", "-l", help="Lister to run", type=click.Choice(SUPPORTED_LISTERS) ) @click.option( "--priority", "-p", default="high", type=click.Choice(["high", "medium", "low"]), help="Task priority for the listed repositories to ingest", ) @click.argument("options", nargs=-1) @click.pass_context def run(ctx, lister, priority, options): from swh.scheduler.cli.utils import parse_options config = deepcopy(ctx.obj["config"]) if options: config.update(parse_options(options)[1]) config["priority"] = priority config["policy"] = "oneshot" get_lister(lister, **config).run() if __name__ == "__main__": lister()