diff --git a/setup.py b/setup.py index fe21851..2406698 100755 --- a/setup.py +++ b/setup.py @@ -1,72 +1,74 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = "requirements-%s.txt" % name else: reqf = "requirements.txt" requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith("#"): continue requirements.append(line) return requirements setup( name="swh.scheduler", description="Software Heritage Scheduler", long_description=long_description, long_description_content_type="text/markdown", python_requires=">=3.7", author="Software Heritage developers", author_email="swh-devel@inria.fr", url="https://forge.softwareheritage.org/diffusion/DSCH/", packages=find_packages(), setup_requires=["setuptools-scm"], use_scm_version=True, install_requires=parse_requirements() + parse_requirements("swh"), extras_require={"testing": parse_requirements("test")}, include_package_data=True, entry_points=""" [console_scripts] swh-scheduler=swh.scheduler.cli:main [swh.cli.subcommands] scheduler=swh.scheduler.cli:cli + [pytest11] + pytest_swh_scheduler=swh.scheduler.pytest_plugin """, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ "Bug Reports": "https://forge.softwareheritage.org/maniphest", "Funding": "https://www.softwareheritage.org/donate", "Source": "https://forge.softwareheritage.org/source/swh-scheduler", "Documentation": "https://docs.softwareheritage.org/devel/swh-scheduler/", }, ) diff --git a/swh/scheduler/pytest_plugin.py b/swh/scheduler/pytest_plugin.py new file mode 100644 index 0000000..cdeb0ff --- /dev/null +++ b/swh/scheduler/pytest_plugin.py @@ -0,0 +1,63 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from datetime import timedelta +import glob +import os + +import pytest + +from swh.core.utils import numfile_sortkey as sortkey + +import swh.scheduler +from swh.scheduler import get_scheduler + + +SQL_DIR = os.path.join(os.path.dirname(swh.scheduler.__file__), "sql") +DUMP_FILES = os.path.join(SQL_DIR, "*.sql") + +# celery tasks for testing purpose; tasks themselves should be +# in swh/scheduler/tests/tasks.py +TASK_NAMES = ["ping", "multiping", "add", "error", "echo"] + + +@pytest.fixture +def swh_scheduler_config(request, postgresql): + scheduler_config = { + "db": postgresql.dsn, + } + + all_dump_files = sorted(glob.glob(DUMP_FILES), key=sortkey) + + cursor = postgresql.cursor() + for fname in all_dump_files: + with open(fname) as fobj: + cursor.execute(fobj.read()) + postgresql.commit() + + return scheduler_config + + +@pytest.fixture +def swh_scheduler(swh_scheduler_config): + scheduler = get_scheduler("local", swh_scheduler_config) + for taskname in TASK_NAMES: + scheduler.create_task_type( + { + "type": "swh-test-{}".format(taskname), + "description": "The {} testing task".format(taskname), + "backend_name": "swh.scheduler.tests.tasks.{}".format(taskname), + "default_interval": timedelta(days=1), + "min_interval": timedelta(hours=6), + "max_interval": timedelta(days=12), + } + ) + + return scheduler + + +# this alias is used to be able to easily instantiate a db-backed Scheduler +# eg. for the RPC client/server test suite. +swh_db_scheduler = swh_scheduler diff --git a/swh/scheduler/tests/__init__.py b/swh/scheduler/tests/__init__.py index 7f3771d..e69de29 100644 --- a/swh/scheduler/tests/__init__.py +++ b/swh/scheduler/tests/__init__.py @@ -1,5 +0,0 @@ -from os import path -import swh.scheduler - - -SQL_DIR = path.join(path.dirname(swh.scheduler.__file__), "sql") diff --git a/swh/scheduler/tests/conftest.py b/swh/scheduler/tests/conftest.py index 9aeb208..8acc02e 100644 --- a/swh/scheduler/tests/conftest.py +++ b/swh/scheduler/tests/conftest.py @@ -1,134 +1,84 @@ # Copyright (C) 2016-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import pytest -import glob -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone import pkg_resources from typing import List -from swh.core.utils import numfile_sortkey as sortkey -from swh.scheduler import get_scheduler -from swh.scheduler.tests import SQL_DIR from swh.scheduler.model import ListedOrigin, Lister from swh.scheduler.tests.common import LISTERS # make sure we are not fooled by CELERY_ config environment vars for var in [x for x in os.environ.keys() if x.startswith("CELERY")]: os.environ.pop(var) # test_cli tests depends on a en/C locale, so ensure it os.environ["LC_ALL"] = "C.UTF-8" -DUMP_FILES = os.path.join(SQL_DIR, "*.sql") - -# celery tasks for testing purpose; tasks themselves should be -# in swh/scheduler/tests/tasks.py -TASK_NAMES = ["ping", "multiping", "add", "error", "echo"] - @pytest.fixture(scope="session") def celery_enable_logging(): return True @pytest.fixture(scope="session") def celery_includes(): task_modules = [ "swh.scheduler.tests.tasks", ] for entrypoint in pkg_resources.iter_entry_points("swh.workers"): task_modules.extend(entrypoint.load()().get("task_modules", [])) return task_modules @pytest.fixture(scope="session") def celery_parameters(): return { "task_cls": "swh.scheduler.task:SWHTask", } @pytest.fixture(scope="session") def celery_config(): return { "accept_content": ["application/x-msgpack", "application/json"], "task_serializer": "msgpack", "result_serializer": "json", } # use the celery_session_app fixture to monkeypatch the 'main' # swh.scheduler.celery_backend.config.app Celery application # with the test application @pytest.fixture(scope="session") def swh_app(celery_session_app): from swh.scheduler.celery_backend import config config.app = celery_session_app yield celery_session_app -@pytest.fixture -def swh_scheduler_config(request, postgresql): - scheduler_config = { - "db": postgresql.dsn, - } - - all_dump_files = sorted(glob.glob(DUMP_FILES), key=sortkey) - - cursor = postgresql.cursor() - for fname in all_dump_files: - with open(fname) as fobj: - cursor.execute(fobj.read()) - postgresql.commit() - - return scheduler_config - - -@pytest.fixture -def swh_scheduler(swh_scheduler_config): - scheduler = get_scheduler("local", swh_scheduler_config) - for taskname in TASK_NAMES: - scheduler.create_task_type( - { - "type": "swh-test-{}".format(taskname), - "description": "The {} testing task".format(taskname), - "backend_name": "swh.scheduler.tests.tasks.{}".format(taskname), - "default_interval": timedelta(days=1), - "min_interval": timedelta(hours=6), - "max_interval": timedelta(days=12), - } - ) - - return scheduler - - -# this alias is used to be able to easily instantiate a db-backed Scheduler -# eg. for the RPC client/server test suite. -swh_db_scheduler = swh_scheduler - - @pytest.fixture def stored_lister(swh_scheduler) -> Lister: """Store a lister in the scheduler and return its information""" return swh_scheduler.get_or_create_lister(**LISTERS[0]) @pytest.fixture def listed_origins(stored_lister) -> List[ListedOrigin]: """Return a (fixed) set of 1000 listed origins""" return [ ListedOrigin( lister_id=stored_lister.id, url=f"https://example.com/{i:04d}.git", visit_type="git", last_update=datetime(2020, 6, 15, 16, 0, 0, i, tzinfo=timezone.utc), ) for i in range(1000) ]