diff --git a/PKG-INFO b/PKG-INFO index 3508181..45b710f 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,30 +1,30 @@ Metadata-Version: 2.1 Name: swh.scheduler -Version: 0.5.3 +Version: 0.6.0 Summary: Software Heritage Scheduler Home-page: https://forge.softwareheritage.org/diffusion/DSCH/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-scheduler Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-scheduler/ Description: swh-scheduler ============= Job scheduler for the Software Heritage project. Task manager for asynchronous/delayed tasks, used for both recurrent (e.g., listing a forge, loading new stuff from a Git repository) and one-off activities (e.g., loading a specific version of a source package). Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing diff --git a/requirements-swh.txt b/requirements-swh.txt index cfb397d..b8b4b27 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,2 +1,2 @@ -swh.core[db,http] >= 0.0.65 +swh.core[db,http] >= 0.3 swh.storage >= 0.11.1 diff --git a/setup.py b/setup.py index ec39d15..b216dce 100755 --- a/setup.py +++ b/setup.py @@ -1,72 +1,70 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from io import open from os import path from setuptools import find_packages, setup here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = "requirements-%s.txt" % name else: reqf = "requirements.txt" requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith("#"): continue requirements.append(line) return requirements setup( name="swh.scheduler", description="Software Heritage Scheduler", long_description=long_description, long_description_content_type="text/markdown", python_requires=">=3.7", author="Software Heritage developers", author_email="swh-devel@inria.fr", url="https://forge.softwareheritage.org/diffusion/DSCH/", packages=find_packages(), setup_requires=["setuptools-scm"], use_scm_version=True, install_requires=parse_requirements() + parse_requirements("swh"), extras_require={"testing": parse_requirements("test")}, include_package_data=True, entry_points=""" - [console_scripts] - swh-scheduler=swh.scheduler.cli:main [swh.cli.subcommands] - scheduler=swh.scheduler.cli:cli + scheduler=swh.scheduler.cli """, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ "Bug Reports": "https://forge.softwareheritage.org/maniphest", "Funding": "https://www.softwareheritage.org/donate", "Source": "https://forge.softwareheritage.org/source/swh-scheduler", "Documentation": "https://docs.softwareheritage.org/devel/swh-scheduler/", }, ) diff --git a/swh.scheduler.egg-info/PKG-INFO b/swh.scheduler.egg-info/PKG-INFO index 3508181..45b710f 100644 --- a/swh.scheduler.egg-info/PKG-INFO +++ b/swh.scheduler.egg-info/PKG-INFO @@ -1,30 +1,30 @@ Metadata-Version: 2.1 Name: swh.scheduler -Version: 0.5.3 +Version: 0.6.0 Summary: Software Heritage Scheduler Home-page: https://forge.softwareheritage.org/diffusion/DSCH/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-scheduler Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-scheduler/ Description: swh-scheduler ============= Job scheduler for the Software Heritage project. Task manager for asynchronous/delayed tasks, used for both recurrent (e.g., listing a forge, loading new stuff from a Git repository) and one-off activities (e.g., loading a specific version of a source package). Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing diff --git a/swh.scheduler.egg-info/SOURCES.txt b/swh.scheduler.egg-info/SOURCES.txt index dee8358..3fce2f7 100644 --- a/swh.scheduler.egg-info/SOURCES.txt +++ b/swh.scheduler.egg-info/SOURCES.txt @@ -1,105 +1,105 @@ .gitignore .pre-commit-config.yaml AUTHORS CODE_OF_CONDUCT.md CONTRIBUTORS LICENSE LICENSE.Celery MANIFEST.in Makefile README.md conftest.py mypy.ini pyproject.toml pytest.ini requirements-swh.txt requirements-test.txt requirements.txt setup.cfg setup.py tox.ini data/README.md data/elastic-template.json data/update-index-settings.json docs/.gitignore docs/Makefile docs/conf.py docs/index.rst docs/_static/.placeholder docs/_templates/.placeholder sql/.gitignore sql/Makefile sql/updates/02.sql sql/updates/03.sql sql/updates/04.sql sql/updates/05.sql sql/updates/06.sql sql/updates/07.sql sql/updates/08.sql sql/updates/09.sql sql/updates/10.sql sql/updates/11.sql sql/updates/12.sql sql/updates/13.sql sql/updates/14.sql sql/updates/15.sql sql/updates/16.sql sql/updates/17.sql swh/__init__.py swh.scheduler.egg-info/PKG-INFO swh.scheduler.egg-info/SOURCES.txt swh.scheduler.egg-info/dependency_links.txt swh.scheduler.egg-info/entry_points.txt swh.scheduler.egg-info/requires.txt swh.scheduler.egg-info/top_level.txt swh/scheduler/__init__.py swh/scheduler/backend.py swh/scheduler/backend_es.py swh/scheduler/cli_utils.py swh/scheduler/elasticsearch_memory.py swh/scheduler/exc.py swh/scheduler/interface.py swh/scheduler/model.py swh/scheduler/py.typed swh/scheduler/pytest_plugin.py swh/scheduler/task.py swh/scheduler/utils.py swh/scheduler/api/__init__.py swh/scheduler/api/client.py swh/scheduler/api/serializers.py swh/scheduler/api/server.py swh/scheduler/celery_backend/__init__.py swh/scheduler/celery_backend/config.py swh/scheduler/celery_backend/listener.py swh/scheduler/celery_backend/pika_listener.py swh/scheduler/celery_backend/runner.py swh/scheduler/cli/__init__.py swh/scheduler/cli/admin.py swh/scheduler/cli/celery_monitor.py swh/scheduler/cli/task.py swh/scheduler/cli/task_type.py swh/scheduler/cli/utils.py -swh/scheduler/sql/10-swh-init.sql -swh/scheduler/sql/30-swh-schema.sql -swh/scheduler/sql/40-swh-func.sql -swh/scheduler/sql/50-swh-data.sql -swh/scheduler/sql/60-swh-indexes.sql +swh/scheduler/sql/10-superuser-init.sql +swh/scheduler/sql/30-schema.sql +swh/scheduler/sql/40-func.sql +swh/scheduler/sql/50-data.sql +swh/scheduler/sql/60-indexes.sql swh/scheduler/tests/__init__.py swh/scheduler/tests/common.py swh/scheduler/tests/conftest.py swh/scheduler/tests/tasks.py swh/scheduler/tests/test_api_client.py swh/scheduler/tests/test_celery_tasks.py swh/scheduler/tests/test_cli.py swh/scheduler/tests/test_cli_celery_monitor.py swh/scheduler/tests/test_cli_task_type.py swh/scheduler/tests/test_common.py swh/scheduler/tests/test_model.py swh/scheduler/tests/test_scheduler.py swh/scheduler/tests/test_server.py swh/scheduler/tests/test_utils.py swh/scheduler/tests/es/__init__.py swh/scheduler/tests/es/conftest.py swh/scheduler/tests/es/test_backend_es.py swh/scheduler/tests/es/test_cli_task.py swh/scheduler/tests/es/test_elasticsearch_memory.py \ No newline at end of file diff --git a/swh.scheduler.egg-info/entry_points.txt b/swh.scheduler.egg-info/entry_points.txt index 85127ac..7f54aac 100644 --- a/swh.scheduler.egg-info/entry_points.txt +++ b/swh.scheduler.egg-info/entry_points.txt @@ -1,6 +1,4 @@ - [console_scripts] - swh-scheduler=swh.scheduler.cli:main [swh.cli.subcommands] - scheduler=swh.scheduler.cli:cli + scheduler=swh.scheduler.cli \ No newline at end of file diff --git a/swh.scheduler.egg-info/requires.txt b/swh.scheduler.egg-info/requires.txt index 66c31ce..483d6bb 100644 --- a/swh.scheduler.egg-info/requires.txt +++ b/swh.scheduler.egg-info/requires.txt @@ -1,22 +1,22 @@ arrow attrs attrs-strict celery>=4.3 Click elasticsearch>5.4 flask pika>=1.1.0 psycopg2 pyyaml vcversioner setuptools -swh.core[db,http]>=0.0.65 +swh.core[db,http]>=0.3 swh.storage>=0.11.1 [testing] pytest pytest-mock pytest-postgresql>=2.1.0 celery>=4.3 hypothesis>=3.11.0 swh.lister diff --git a/swh/scheduler/cli/__init__.py b/swh/scheduler/cli/__init__.py index eb4db7c..2343f79 100644 --- a/swh/scheduler/cli/__init__.py +++ b/swh/scheduler/cli/__init__.py @@ -1,93 +1,95 @@ # Copyright (C) 2016-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information # WARNING: do not import unnecessary things here to keep cli startup time under # control import logging import click -from swh.core.cli import CONTEXT_SETTINGS, AliasedGroup +from swh.core.cli import CONTEXT_SETTINGS, AliasedGroup, swh as swh_cli_group -@click.group(name="scheduler", context_settings=CONTEXT_SETTINGS, cls=AliasedGroup) +@swh_cli_group.group( + name="scheduler", context_settings=CONTEXT_SETTINGS, cls=AliasedGroup +) @click.option( "--config-file", "-C", default=None, type=click.Path(exists=True, dir_okay=False,), help="Configuration file.", ) @click.option( "--database", "-d", default=None, help="Scheduling database DSN (imply cls is 'local')", ) @click.option( "--url", "-u", default=None, help="Scheduler's url access (imply cls is 'remote')" ) @click.option( "--no-stdout", is_flag=True, default=False, help="Do NOT output logs on the console" ) @click.pass_context def cli(ctx, config_file, database, url, no_stdout): """Software Heritage Scheduler tools. Use a local scheduler instance by default (plugged to the main scheduler db). """ try: from psycopg2 import OperationalError except ImportError: class OperationalError(Exception): pass from swh.core import config from swh.scheduler import DEFAULT_CONFIG, get_scheduler ctx.ensure_object(dict) logger = logging.getLogger(__name__) scheduler = None conf = config.read(config_file, DEFAULT_CONFIG) if "scheduler" not in conf: raise ValueError("missing 'scheduler' configuration") if database: conf["scheduler"]["cls"] = "local" conf["scheduler"]["args"]["db"] = database elif url: conf["scheduler"]["cls"] = "remote" conf["scheduler"]["args"] = {"url": url} sched_conf = conf["scheduler"] try: logger.debug("Instantiating scheduler with %s" % (sched_conf)) scheduler = get_scheduler(**sched_conf) except (ValueError, OperationalError): # it's the subcommand to decide whether not having a proper # scheduler instance is a problem. pass ctx.obj["scheduler"] = scheduler ctx.obj["config"] = conf from . import admin, celery_monitor, task, task_type # noqa def main(): import click.core click.core.DEPRECATED_HELP_NOTICE = """ DEPRECATED! Please use the command 'swh scheduler'.""" cli.deprecated = True return cli(auto_envvar_prefix="SWH_SCHEDULER") if __name__ == "__main__": main() diff --git a/swh/scheduler/sql/10-swh-init.sql b/swh/scheduler/sql/10-superuser-init.sql similarity index 100% rename from swh/scheduler/sql/10-swh-init.sql rename to swh/scheduler/sql/10-superuser-init.sql diff --git a/swh/scheduler/sql/30-swh-schema.sql b/swh/scheduler/sql/30-schema.sql similarity index 100% rename from swh/scheduler/sql/30-swh-schema.sql rename to swh/scheduler/sql/30-schema.sql diff --git a/swh/scheduler/sql/40-swh-func.sql b/swh/scheduler/sql/40-func.sql similarity index 100% rename from swh/scheduler/sql/40-swh-func.sql rename to swh/scheduler/sql/40-func.sql diff --git a/swh/scheduler/sql/50-swh-data.sql b/swh/scheduler/sql/50-data.sql similarity index 100% rename from swh/scheduler/sql/50-swh-data.sql rename to swh/scheduler/sql/50-data.sql diff --git a/swh/scheduler/sql/60-swh-indexes.sql b/swh/scheduler/sql/60-indexes.sql similarity index 100% rename from swh/scheduler/sql/60-swh-indexes.sql rename to swh/scheduler/sql/60-indexes.sql