diff --git a/PKG-INFO b/PKG-INFO index 62bc8bf..42ee063 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,30 +1,30 @@ Metadata-Version: 2.1 Name: swh.scheduler -Version: 0.3.0 +Version: 0.4.0 Summary: Software Heritage Scheduler Home-page: https://forge.softwareheritage.org/diffusion/DSCH/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-scheduler Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-scheduler/ Description: swh-scheduler ============= Job scheduler for the Software Heritage project. Task manager for asynchronous/delayed tasks, used for both recurrent (e.g., listing a forge, loading new stuff from a Git repository) and one-off activities (e.g., loading a specific version of a source package). Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing diff --git a/setup.py b/setup.py index fe21851..2406698 100755 --- a/setup.py +++ b/setup.py @@ -1,72 +1,74 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = "requirements-%s.txt" % name else: reqf = "requirements.txt" requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith("#"): continue requirements.append(line) return requirements setup( name="swh.scheduler", description="Software Heritage Scheduler", long_description=long_description, long_description_content_type="text/markdown", python_requires=">=3.7", author="Software Heritage developers", author_email="swh-devel@inria.fr", url="https://forge.softwareheritage.org/diffusion/DSCH/", packages=find_packages(), setup_requires=["setuptools-scm"], use_scm_version=True, install_requires=parse_requirements() + parse_requirements("swh"), extras_require={"testing": parse_requirements("test")}, include_package_data=True, entry_points=""" [console_scripts] swh-scheduler=swh.scheduler.cli:main [swh.cli.subcommands] scheduler=swh.scheduler.cli:cli + [pytest11] + pytest_swh_scheduler=swh.scheduler.pytest_plugin """, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ "Bug Reports": "https://forge.softwareheritage.org/maniphest", "Funding": "https://www.softwareheritage.org/donate", "Source": "https://forge.softwareheritage.org/source/swh-scheduler", "Documentation": "https://docs.softwareheritage.org/devel/swh-scheduler/", }, ) diff --git a/swh.scheduler.egg-info/PKG-INFO b/swh.scheduler.egg-info/PKG-INFO index 62bc8bf..42ee063 100644 --- a/swh.scheduler.egg-info/PKG-INFO +++ b/swh.scheduler.egg-info/PKG-INFO @@ -1,30 +1,30 @@ Metadata-Version: 2.1 Name: swh.scheduler -Version: 0.3.0 +Version: 0.4.0 Summary: Software Heritage Scheduler Home-page: https://forge.softwareheritage.org/diffusion/DSCH/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-scheduler Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-scheduler/ Description: swh-scheduler ============= Job scheduler for the Software Heritage project. Task manager for asynchronous/delayed tasks, used for both recurrent (e.g., listing a forge, loading new stuff from a Git repository) and one-off activities (e.g., loading a specific version of a source package). Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing diff --git a/swh.scheduler.egg-info/SOURCES.txt b/swh.scheduler.egg-info/SOURCES.txt index f374f17..26ef057 100644 --- a/swh.scheduler.egg-info/SOURCES.txt +++ b/swh.scheduler.egg-info/SOURCES.txt @@ -1,105 +1,106 @@ .gitignore .pre-commit-config.yaml AUTHORS CODE_OF_CONDUCT.md CONTRIBUTORS LICENSE LICENSE.Celery MANIFEST.in Makefile README.md conftest.py mypy.ini pyproject.toml pytest.ini requirements-swh.txt requirements-test.txt requirements.txt setup.cfg setup.py tox.ini version.txt data/README.md data/elastic-template.json data/update-index-settings.json docs/.gitignore docs/Makefile docs/conf.py docs/index.rst docs/_static/.placeholder docs/_templates/.placeholder sql/.gitignore sql/Makefile sql/updates/02.sql sql/updates/03.sql sql/updates/04.sql sql/updates/05.sql sql/updates/06.sql sql/updates/07.sql sql/updates/08.sql sql/updates/09.sql sql/updates/10.sql sql/updates/11.sql sql/updates/12.sql sql/updates/13.sql sql/updates/14.sql sql/updates/15.sql sql/updates/16.sql sql/updates/17.sql swh/__init__.py swh.scheduler.egg-info/PKG-INFO swh.scheduler.egg-info/SOURCES.txt swh.scheduler.egg-info/dependency_links.txt swh.scheduler.egg-info/entry_points.txt swh.scheduler.egg-info/requires.txt swh.scheduler.egg-info/top_level.txt swh/scheduler/__init__.py swh/scheduler/backend.py swh/scheduler/backend_es.py swh/scheduler/cli_utils.py swh/scheduler/elasticsearch_memory.py swh/scheduler/exc.py swh/scheduler/interface.py swh/scheduler/model.py swh/scheduler/py.typed +swh/scheduler/pytest_plugin.py swh/scheduler/task.py swh/scheduler/utils.py swh/scheduler/api/__init__.py swh/scheduler/api/client.py swh/scheduler/api/serializers.py swh/scheduler/api/server.py swh/scheduler/celery_backend/__init__.py swh/scheduler/celery_backend/config.py swh/scheduler/celery_backend/listener.py swh/scheduler/celery_backend/pika_listener.py swh/scheduler/celery_backend/runner.py swh/scheduler/cli/__init__.py swh/scheduler/cli/admin.py swh/scheduler/cli/celery_monitor.py swh/scheduler/cli/task.py swh/scheduler/cli/task_type.py swh/scheduler/cli/utils.py swh/scheduler/sql/10-swh-init.sql swh/scheduler/sql/30-swh-schema.sql swh/scheduler/sql/40-swh-func.sql swh/scheduler/sql/50-swh-data.sql swh/scheduler/sql/60-swh-indexes.sql swh/scheduler/tests/__init__.py swh/scheduler/tests/common.py swh/scheduler/tests/conftest.py swh/scheduler/tests/tasks.py swh/scheduler/tests/test_api_client.py swh/scheduler/tests/test_celery_tasks.py swh/scheduler/tests/test_cli.py swh/scheduler/tests/test_cli_celery_monitor.py swh/scheduler/tests/test_cli_task_type.py swh/scheduler/tests/test_common.py swh/scheduler/tests/test_model.py swh/scheduler/tests/test_scheduler.py swh/scheduler/tests/test_server.py swh/scheduler/tests/test_utils.py swh/scheduler/tests/es/__init__.py swh/scheduler/tests/es/conftest.py swh/scheduler/tests/es/test_backend_es.py swh/scheduler/tests/es/test_cli_task.py swh/scheduler/tests/es/test_elasticsearch_memory.py \ No newline at end of file diff --git a/swh.scheduler.egg-info/entry_points.txt b/swh.scheduler.egg-info/entry_points.txt index 85127ac..1c32d90 100644 --- a/swh.scheduler.egg-info/entry_points.txt +++ b/swh.scheduler.egg-info/entry_points.txt @@ -1,6 +1,8 @@ [console_scripts] swh-scheduler=swh.scheduler.cli:main [swh.cli.subcommands] scheduler=swh.scheduler.cli:cli + [pytest11] + pytest_swh_scheduler=swh.scheduler.pytest_plugin \ No newline at end of file diff --git a/swh/scheduler/pytest_plugin.py b/swh/scheduler/pytest_plugin.py new file mode 100644 index 0000000..cdeb0ff --- /dev/null +++ b/swh/scheduler/pytest_plugin.py @@ -0,0 +1,63 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from datetime import timedelta +import glob +import os + +import pytest + +from swh.core.utils import numfile_sortkey as sortkey + +import swh.scheduler +from swh.scheduler import get_scheduler + + +SQL_DIR = os.path.join(os.path.dirname(swh.scheduler.__file__), "sql") +DUMP_FILES = os.path.join(SQL_DIR, "*.sql") + +# celery tasks for testing purpose; tasks themselves should be +# in swh/scheduler/tests/tasks.py +TASK_NAMES = ["ping", "multiping", "add", "error", "echo"] + + +@pytest.fixture +def swh_scheduler_config(request, postgresql): + scheduler_config = { + "db": postgresql.dsn, + } + + all_dump_files = sorted(glob.glob(DUMP_FILES), key=sortkey) + + cursor = postgresql.cursor() + for fname in all_dump_files: + with open(fname) as fobj: + cursor.execute(fobj.read()) + postgresql.commit() + + return scheduler_config + + +@pytest.fixture +def swh_scheduler(swh_scheduler_config): + scheduler = get_scheduler("local", swh_scheduler_config) + for taskname in TASK_NAMES: + scheduler.create_task_type( + { + "type": "swh-test-{}".format(taskname), + "description": "The {} testing task".format(taskname), + "backend_name": "swh.scheduler.tests.tasks.{}".format(taskname), + "default_interval": timedelta(days=1), + "min_interval": timedelta(hours=6), + "max_interval": timedelta(days=12), + } + ) + + return scheduler + + +# this alias is used to be able to easily instantiate a db-backed Scheduler +# eg. for the RPC client/server test suite. +swh_db_scheduler = swh_scheduler diff --git a/swh/scheduler/tests/__init__.py b/swh/scheduler/tests/__init__.py index 7f3771d..e69de29 100644 --- a/swh/scheduler/tests/__init__.py +++ b/swh/scheduler/tests/__init__.py @@ -1,5 +0,0 @@ -from os import path -import swh.scheduler - - -SQL_DIR = path.join(path.dirname(swh.scheduler.__file__), "sql") diff --git a/swh/scheduler/tests/conftest.py b/swh/scheduler/tests/conftest.py index 9aeb208..8acc02e 100644 --- a/swh/scheduler/tests/conftest.py +++ b/swh/scheduler/tests/conftest.py @@ -1,134 +1,84 @@ # Copyright (C) 2016-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import pytest -import glob -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone import pkg_resources from typing import List -from swh.core.utils import numfile_sortkey as sortkey -from swh.scheduler import get_scheduler -from swh.scheduler.tests import SQL_DIR from swh.scheduler.model import ListedOrigin, Lister from swh.scheduler.tests.common import LISTERS # make sure we are not fooled by CELERY_ config environment vars for var in [x for x in os.environ.keys() if x.startswith("CELERY")]: os.environ.pop(var) # test_cli tests depends on a en/C locale, so ensure it os.environ["LC_ALL"] = "C.UTF-8" -DUMP_FILES = os.path.join(SQL_DIR, "*.sql") - -# celery tasks for testing purpose; tasks themselves should be -# in swh/scheduler/tests/tasks.py -TASK_NAMES = ["ping", "multiping", "add", "error", "echo"] - @pytest.fixture(scope="session") def celery_enable_logging(): return True @pytest.fixture(scope="session") def celery_includes(): task_modules = [ "swh.scheduler.tests.tasks", ] for entrypoint in pkg_resources.iter_entry_points("swh.workers"): task_modules.extend(entrypoint.load()().get("task_modules", [])) return task_modules @pytest.fixture(scope="session") def celery_parameters(): return { "task_cls": "swh.scheduler.task:SWHTask", } @pytest.fixture(scope="session") def celery_config(): return { "accept_content": ["application/x-msgpack", "application/json"], "task_serializer": "msgpack", "result_serializer": "json", } # use the celery_session_app fixture to monkeypatch the 'main' # swh.scheduler.celery_backend.config.app Celery application # with the test application @pytest.fixture(scope="session") def swh_app(celery_session_app): from swh.scheduler.celery_backend import config config.app = celery_session_app yield celery_session_app -@pytest.fixture -def swh_scheduler_config(request, postgresql): - scheduler_config = { - "db": postgresql.dsn, - } - - all_dump_files = sorted(glob.glob(DUMP_FILES), key=sortkey) - - cursor = postgresql.cursor() - for fname in all_dump_files: - with open(fname) as fobj: - cursor.execute(fobj.read()) - postgresql.commit() - - return scheduler_config - - -@pytest.fixture -def swh_scheduler(swh_scheduler_config): - scheduler = get_scheduler("local", swh_scheduler_config) - for taskname in TASK_NAMES: - scheduler.create_task_type( - { - "type": "swh-test-{}".format(taskname), - "description": "The {} testing task".format(taskname), - "backend_name": "swh.scheduler.tests.tasks.{}".format(taskname), - "default_interval": timedelta(days=1), - "min_interval": timedelta(hours=6), - "max_interval": timedelta(days=12), - } - ) - - return scheduler - - -# this alias is used to be able to easily instantiate a db-backed Scheduler -# eg. for the RPC client/server test suite. -swh_db_scheduler = swh_scheduler - - @pytest.fixture def stored_lister(swh_scheduler) -> Lister: """Store a lister in the scheduler and return its information""" return swh_scheduler.get_or_create_lister(**LISTERS[0]) @pytest.fixture def listed_origins(stored_lister) -> List[ListedOrigin]: """Return a (fixed) set of 1000 listed origins""" return [ ListedOrigin( lister_id=stored_lister.id, url=f"https://example.com/{i:04d}.git", visit_type="git", last_update=datetime(2020, 6, 15, 16, 0, 0, i, tzinfo=timezone.utc), ) for i in range(1000) ]