diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec7bad1..e9b4304 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,50 +1,43 @@ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 - hooks: - - id: trailing-whitespace - - id: check-json - - id: check-yaml + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: trailing-whitespace + - id: check-json + - id: check-yaml -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.3 - hooks: - - id: flake8 + - repo: https://gitlab.com/pycqa/flake8 + rev: 4.0.1 + hooks: + - id: flake8 -- repo: https://github.com/codespell-project/codespell - rev: v1.16.0 - hooks: - - id: codespell - args: [-L simpy, -L hist] + - repo: https://github.com/codespell-project/codespell + rev: v2.1.0 + hooks: + - id: codespell + name: Check source code spelling + args: [-L simpy, -L hist] + stages: [commit] + - id: codespell + name: Check commit message spelling + stages: [commit-msg] -- repo: local - hooks: - - id: mypy - name: mypy - entry: mypy - args: [swh] - pass_filenames: false - language: system - types: [python] + - repo: local + hooks: + - id: mypy + name: mypy + entry: mypy + args: [swh] + pass_filenames: false + language: system + types: [python] -- repo: https://github.com/PyCQA/isort - rev: 5.5.2 - hooks: - - id: isort - -- repo: https://github.com/python/black - rev: 19.10b0 - hooks: - - id: black - -# unfortunately, we are far from being able to enable this... -# - repo: https://github.com/PyCQA/pydocstyle.git -# rev: 4.0.0 -# hooks: -# - id: pydocstyle -# name: pydocstyle -# description: pydocstyle is a static analysis tool for checking compliance with Python docstring conventions. -# entry: pydocstyle --convention=google -# language: python -# types: [python] + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort + - repo: https://github.com/python/black + rev: 19.10b0 + hooks: + - id: black diff --git a/PKG-INFO b/PKG-INFO index 381adf0..fb83a0a 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,37 +1,37 @@ Metadata-Version: 2.1 Name: swh.scheduler -Version: 0.23.0 +Version: 1.0.0 Summary: Software Heritage Scheduler Home-page: https://forge.softwareheritage.org/diffusion/DSCH/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-scheduler Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-scheduler/ Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing Provides-Extra: journal Provides-Extra: simulator License-File: LICENSE License-File: LICENSE.Celery License-File: AUTHORS swh-scheduler ============= Job scheduler for the Software Heritage project. Task manager for asynchronous/delayed tasks, used for both recurrent (e.g., listing a forge, loading new stuff from a Git repository) and one-off activities (e.g., loading a specific version of a source package). diff --git a/docs/index.rst b/docs/index.rst index bf185cb..68f8272 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,170 +1,170 @@ .. _swh-scheduler: Software Heritage - Job scheduler ================================= Task manager for asynchronous/delayed tasks, used for both recurrent (e.g., listing a forge, loading new stuff from a Git repository) and one-off activities (e.g., loading a specific version of a source package). Description ----------- This module provides a scheduler service for the Software Heritage platform. It allows to define tasks with a number of properties. In this documentation, we will call these swh-tasks to prevent confusion. These swh-tasks are stored in a database, and a HTTP-based RPC service is provided to create or find existing swh-task declarations. The execution model for these swh-tasks is using Celery. Thus, each swh-task type defined in the database must have a (series of) celery worker capable of executing such a swh-task. Then a number of services are also provided to manage the scheduling of these swh-tasks as Celery tasks. -The `scheduler-runner` service is a daemon that regularly looks for swh-tasks +The ``scheduler-runner`` service is a daemon that regularly looks for swh-tasks in the database that should be scheduled. For each of the selected swh-task, a Celery task is instantiated. -The `scheduler-listener` service is a daemon that listen to the Celery event +The ``scheduler-listener`` service is a daemon that listen to the Celery event bus and maintain scheduled swh-tasks workflow status. SWH Task Model ~~~~~~~~~~~~~~ Each swh-task-type is the declaration of a type of swh-task. Each swh-task-type have the following fields: -- `type`: Name of the swh-task type; can be anything but must be unique, -- `description`: Human-readable task description -- `backend_name`: Name of the task in the job-running backend, -- `default_interval`: Default interval for newly scheduled tasks, -- `min_interval`: Minimum interval between two runs of a task, -- `max_interval`: Maximum interval between two runs of a task, -- `backoff_factor`: Adjustment factor for the backoff between two task runs, -- `max_queue_length`: Maximum length of the queue for this type of tasks, -- `num_retries`: Default number of retries on transient failures, -- `retry_delay`: Retry delay for the task, - -Existing swh-task-types can be listed using the `swh scheduler` command line +- ``type``: Name of the swh-task type; can be anything but must be unique, +- ``description``: Human-readable task description +- ``backend_name``: Name of the task in the job-running backend, +- ``default_interval``: Default interval for newly scheduled tasks, +- ``min_interval``: Minimum interval between two runs of a task, +- ``max_interval``: Maximum interval between two runs of a task, +- ``backoff_factor``: Adjustment factor for the backoff between two task runs, +- ``max_queue_length``: Maximum length of the queue for this type of tasks, +- ``num_retries``: Default number of retries on transient failures, +- ``retry_delay``: Retry delay for the task, + +Existing swh-task-types can be listed using the ``swh scheduler`` command line tool:: $ swh scheduler task-type list Known task types: check-deposit: Pre-checking deposit step before loading into swh archive index-fossology-license: Fossology license indexer task load-git: Update an origin of type git load-hg: Update an origin of type mercurial You can see the details of a swh-task-type:: $ swh scheduler task-type list -v -t load-git Known task types: load-git: swh.loader.git.tasks.UpdateGitRepository Update an origin of type git interval: 64 days, 0:00:00 [12:00:00, 64 days, 0:00:00] backoff_factor: 2.0 max_queue_length: 5000 num_retries: None retry_delay: None An swh-task is an 'instance' of such a swh-task-type, and consists in: -- `arguments`: Arguments passed to the underlying job scheduler, -- `next_run`: Next run of this task should be run on or after that time, -- `current_interval`: Interval between two runs of this task, taking into +- ``arguments``: Arguments passed to the underlying job scheduler, +- ``next_run``: Next run of this task should be run on or after that time, +- ``current_interval``: Interval between two runs of this task, taking into account the backoff factor, -- `policy`: Whether the task is "one-shot" or "recurring", -- `retries_left`: Number of "short delay" retries of the task in case of +- ``policy``: Whether the task is "one-shot" or "recurring", +- ``retries_left``: Number of "short delay" retries of the task in case of transient failure, -- `priority`: Priority of the task, -- `id`: Internal task identifier, -- `type`: References task_type table, -- `status`: Task status ( among "next_run_not_scheduled", "next_run_scheduled", +- ``priority``: Priority of the task, +- ``id``: Internal task identifier, +- ``type``: References task_type table, +- ``status``: Task status ( among "next_run_not_scheduled", "next_run_scheduled", "completed", "disabled"). So a swh-task consist basically in: - a set of parameters defining how the scheduling of the swh-task is handled, - a set of parameters to specify the retry policy in case of transient failure upon execution, -- a set of parameters that defines the job to be done (`bakend_name` + - `arguments`). +- a set of parameters that defines the job to be done (``bakend_name`` + + ``arguments``). You can list pending swh-tasks (tasks that are to be scheduled ASAP):: $ swh scheduler task list-pending load-git --limit 2 Found 1 load-git tasks Task 9052257 Next run: 15 days ago (2019-06-25 10:35:10+00:00) Interval: 2 days, 0:00:00 Type: load-git Policy: recurring Args: 'https://github.com/turtl/mobile' Keyword args: Looking for existing swh-task can be done via the command line tool:: $ swh scheduler task list -t load-hg --limit 2 Found 2 tasks Task 168802702 Next run: in 4 hours (2019-07-10 17:56:48+00:00) Interval: 1 day, 0:00:00 Type: load-hg Policy: recurring Status: next_run_not_scheduled Priority: Args: 'https://bitbucket.org/kepung/pypy' Keyword args: Task 169800445 Next run: in a month (2019-08-10 17:54:24+00:00) Interval: 32 days, 0:00:00 Type: load-hg Policy: recurring Status: next_run_not_scheduled Priority: Args: 'https://bitbucket.org/lunixbochs/pypy-1' Keyword args: Writing a new worker for a new swh-task-type -------------------------------------------- When you want to add a new swh-task-type, you need a celery worker backend capable of executing this new task-type instances. Celery workers for swh-scheduler based tasks should be started using the Celery -app in `swh.scheduler.celery_config`. This later, among other things, provides +app in ``swh.scheduler.celery_config``. This later, among other things, provides a loading mechanism for task types based on pkg_resources declared plugins under -the `[swh.workers]` entry point. +the ``[swh.workers]`` entry point. TODO: add a fully working example of a dumb task. Reference Documentation ----------------------- .. toctree:: :maxdepth: 2 cli simulator /apidoc/swh.scheduler diff --git a/requirements-swh.txt b/requirements-swh.txt index d8b75bf..b0de19e 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,2 +1,2 @@ -swh.core[db,http] >= 0.14.0 +swh.core[db,http] >= 2 swh.storage >= 0.11.1 diff --git a/requirements-test.txt b/requirements-test.txt index 726caf0..b3b82a1 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,11 +1,11 @@ -pytest +pytest < 7.0.0 # v7.0.0 removed _pytest.tmpdir.TempdirFactory, which is used by some of the pytest plugins we use pytest-mock celery >= 4.3 hypothesis >= 3.11.0 swh.lister swh.storage[testing] types-click types-flask types-pyyaml types-requests types-Deprecated diff --git a/requirements.txt b/requirements.txt index 64c6305..357c6ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ # Add here external Python modules dependencies, one per line. Module names # should match https://pypi.python.org/pypi names. For the full spec or # dependency lines, see https://pip.readthedocs.org/en/1.1/requirements.html attrs attrs-strict celery >= 4.3, != 5.0.3 -click < 8.0 +click flask humanize pika >= 1.1.0 psycopg2 pyyaml requests setuptools typing-extensions diff --git a/swh.scheduler.egg-info/PKG-INFO b/swh.scheduler.egg-info/PKG-INFO index 381adf0..fb83a0a 100644 --- a/swh.scheduler.egg-info/PKG-INFO +++ b/swh.scheduler.egg-info/PKG-INFO @@ -1,37 +1,37 @@ Metadata-Version: 2.1 Name: swh.scheduler -Version: 0.23.0 +Version: 1.0.0 Summary: Software Heritage Scheduler Home-page: https://forge.softwareheritage.org/diffusion/DSCH/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-scheduler Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-scheduler/ Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing Provides-Extra: journal Provides-Extra: simulator License-File: LICENSE License-File: LICENSE.Celery License-File: AUTHORS swh-scheduler ============= Job scheduler for the Software Heritage project. Task manager for asynchronous/delayed tasks, used for both recurrent (e.g., listing a forge, loading new stuff from a Git repository) and one-off activities (e.g., loading a specific version of a source package). diff --git a/swh.scheduler.egg-info/SOURCES.txt b/swh.scheduler.egg-info/SOURCES.txt index 959bfa7..a2910b0 100644 --- a/swh.scheduler.egg-info/SOURCES.txt +++ b/swh.scheduler.egg-info/SOURCES.txt @@ -1,132 +1,133 @@ .gitignore .pre-commit-config.yaml AUTHORS CODE_OF_CONDUCT.md CONTRIBUTORS LICENSE LICENSE.Celery MANIFEST.in Makefile README.md conftest.py mypy.ini pyproject.toml pytest.ini requirements-journal.txt requirements-simulator.txt requirements-swh.txt requirements-test.txt requirements.txt setup.cfg setup.py tox.ini data/README.md data/elastic-template.json data/update-index-settings.json docs/.gitignore docs/Makefile docs/cli.rst docs/conf.py docs/index.rst docs/simulator.rst docs/_static/.placeholder docs/_templates/.placeholder sql/.gitignore sql/Makefile -sql/updates/02.sql -sql/updates/03.sql -sql/updates/04.sql -sql/updates/05.sql -sql/updates/06.sql -sql/updates/07.sql -sql/updates/08.sql -sql/updates/09.sql -sql/updates/10.sql -sql/updates/11.sql -sql/updates/12.sql -sql/updates/13.sql -sql/updates/14.sql -sql/updates/15.sql -sql/updates/16.sql -sql/updates/17.sql -sql/updates/18.sql -sql/updates/19.sql -sql/updates/20.sql -sql/updates/23.sql -sql/updates/24.sql -sql/updates/25.sql -sql/updates/26.sql -sql/updates/27.sql -sql/updates/28.sql -sql/updates/29.sql -sql/updates/30-bis.sql -sql/updates/30.sql -sql/updates/31.sql -sql/updates/32.sql swh/__init__.py swh.scheduler.egg-info/PKG-INFO swh.scheduler.egg-info/SOURCES.txt swh.scheduler.egg-info/dependency_links.txt swh.scheduler.egg-info/entry_points.txt swh.scheduler.egg-info/requires.txt swh.scheduler.egg-info/top_level.txt swh/scheduler/__init__.py swh/scheduler/backend.py swh/scheduler/cli_utils.py swh/scheduler/exc.py swh/scheduler/interface.py swh/scheduler/journal_client.py swh/scheduler/model.py swh/scheduler/py.typed swh/scheduler/pytest_plugin.py swh/scheduler/task.py swh/scheduler/utils.py swh/scheduler/api/__init__.py swh/scheduler/api/client.py swh/scheduler/api/serializers.py swh/scheduler/api/server.py swh/scheduler/celery_backend/__init__.py swh/scheduler/celery_backend/config.py swh/scheduler/celery_backend/pika_listener.py swh/scheduler/celery_backend/recurrent_visits.py swh/scheduler/celery_backend/runner.py swh/scheduler/cli/__init__.py swh/scheduler/cli/admin.py swh/scheduler/cli/celery_monitor.py swh/scheduler/cli/journal.py swh/scheduler/cli/origin.py swh/scheduler/cli/simulator.py swh/scheduler/cli/task.py swh/scheduler/cli/task_type.py swh/scheduler/cli/utils.py swh/scheduler/simulator/__init__.py swh/scheduler/simulator/common.py swh/scheduler/simulator/origin_scheduler.py swh/scheduler/simulator/origins.py swh/scheduler/simulator/task_scheduler.py swh/scheduler/sql/10-superuser-init.sql swh/scheduler/sql/30-schema.sql swh/scheduler/sql/40-func.sql swh/scheduler/sql/50-data.sql swh/scheduler/sql/60-indexes.sql +swh/scheduler/sql/upgrades/02.sql +swh/scheduler/sql/upgrades/03.sql +swh/scheduler/sql/upgrades/04.sql +swh/scheduler/sql/upgrades/05.sql +swh/scheduler/sql/upgrades/06.sql +swh/scheduler/sql/upgrades/07.sql +swh/scheduler/sql/upgrades/08.sql +swh/scheduler/sql/upgrades/09.sql +swh/scheduler/sql/upgrades/10.sql +swh/scheduler/sql/upgrades/11.sql +swh/scheduler/sql/upgrades/12.sql +swh/scheduler/sql/upgrades/13.sql +swh/scheduler/sql/upgrades/14.sql +swh/scheduler/sql/upgrades/15.sql +swh/scheduler/sql/upgrades/16.sql +swh/scheduler/sql/upgrades/17.sql +swh/scheduler/sql/upgrades/18.sql +swh/scheduler/sql/upgrades/19.sql +swh/scheduler/sql/upgrades/20.sql +swh/scheduler/sql/upgrades/23.sql +swh/scheduler/sql/upgrades/24.sql +swh/scheduler/sql/upgrades/25.sql +swh/scheduler/sql/upgrades/26.sql +swh/scheduler/sql/upgrades/27.sql +swh/scheduler/sql/upgrades/28.sql +swh/scheduler/sql/upgrades/29.sql +swh/scheduler/sql/upgrades/30-bis.sql +swh/scheduler/sql/upgrades/30.sql +swh/scheduler/sql/upgrades/31.sql +swh/scheduler/sql/upgrades/32.sql +swh/scheduler/sql/upgrades/33.sql swh/scheduler/tests/__init__.py swh/scheduler/tests/common.py swh/scheduler/tests/conftest.py swh/scheduler/tests/tasks.py swh/scheduler/tests/test_api_client.py swh/scheduler/tests/test_celery_tasks.py swh/scheduler/tests/test_cli.py swh/scheduler/tests/test_cli_celery_monitor.py swh/scheduler/tests/test_cli_journal.py swh/scheduler/tests/test_cli_origin.py swh/scheduler/tests/test_cli_task_type.py swh/scheduler/tests/test_common.py swh/scheduler/tests/test_config.py swh/scheduler/tests/test_init.py swh/scheduler/tests/test_journal_client.py swh/scheduler/tests/test_model.py swh/scheduler/tests/test_recurrent_visits.py swh/scheduler/tests/test_scheduler.py swh/scheduler/tests/test_server.py swh/scheduler/tests/test_simulator.py swh/scheduler/tests/test_utils.py \ No newline at end of file diff --git a/swh.scheduler.egg-info/entry_points.txt b/swh.scheduler.egg-info/entry_points.txt index 85f4ea9..5320682 100644 --- a/swh.scheduler.egg-info/entry_points.txt +++ b/swh.scheduler.egg-info/entry_points.txt @@ -1,5 +1,3 @@ - - [swh.cli.subcommands] - scheduler=swh.scheduler.cli - scheduler-journal=swh.scheduler.cli.journal - \ No newline at end of file +[swh.cli.subcommands] +scheduler = swh.scheduler.cli +scheduler-journal = swh.scheduler.cli.journal diff --git a/swh.scheduler.egg-info/requires.txt b/swh.scheduler.egg-info/requires.txt index 03cdbc7..278bfd6 100644 --- a/swh.scheduler.egg-info/requires.txt +++ b/swh.scheduler.egg-info/requires.txt @@ -1,37 +1,37 @@ attrs attrs-strict celery!=5.0.3,>=4.3 -click<8.0 +click flask humanize pika>=1.1.0 psycopg2 pyyaml requests setuptools typing-extensions -swh.core[db,http]>=0.14.0 +swh.core[db,http]>=2 swh.storage>=0.11.1 [journal] swh.journal [simulator] plotille simpy<4,>=3 [testing] -pytest +pytest<7.0.0 pytest-mock celery>=4.3 hypothesis>=3.11.0 swh.lister swh.storage[testing] types-click types-flask types-pyyaml types-requests types-Deprecated swh.journal plotille simpy<4,>=3 diff --git a/swh/scheduler/__init__.py b/swh/scheduler/__init__.py index b93306d..fbb52cd 100644 --- a/swh/scheduler/__init__.py +++ b/swh/scheduler/__init__.py @@ -1,68 +1,73 @@ # Copyright (C) 2018-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import annotations from importlib import import_module from typing import TYPE_CHECKING, Any, Dict import warnings DEFAULT_CONFIG = { "scheduler": ( "dict", {"cls": "local", "db": "dbname=softwareheritage-scheduler-dev",}, ) } # current configuration. To be set by the config loading mechanism CONFIG = {} # type: Dict[str, Any] if TYPE_CHECKING: from swh.scheduler.interface import SchedulerInterface BACKEND_TYPES: Dict[str, str] = { - "local": ".backend.SchedulerBackend", + "postgresql": ".backend.SchedulerBackend", "remote": ".api.client.RemoteScheduler", + # deprecated + "local": ".backend.SchedulerBackend", } def get_scheduler(cls: str, **kwargs) -> SchedulerInterface: """ Get a scheduler object of class `cls` with arguments `**kwargs`. Args: cls: scheduler's class, either 'local' or 'remote' kwargs: arguments to pass to the class' constructor Returns: an instance of swh.scheduler, either local or remote: local: swh.scheduler.backend.SchedulerBackend remote: swh.scheduler.api.client.RemoteScheduler Raises: ValueError if passed an unknown storage class. """ if "args" in kwargs: warnings.warn( 'Explicit "args" key is deprecated, use keys directly instead.', DeprecationWarning, ) kwargs = kwargs["args"] class_path = BACKEND_TYPES.get(cls) if class_path is None: raise ValueError( f"Unknown Scheduler class `{cls}`. " f"Supported: {', '.join(BACKEND_TYPES)}" ) (module_path, class_name) = class_path.rsplit(".", 1) module = import_module(module_path, package=__package__) BackendClass = getattr(module, class_name) return BackendClass(**kwargs) + + +get_datastore = get_scheduler diff --git a/swh/scheduler/backend.py b/swh/scheduler/backend.py index e3850d4..0b3c3f8 100644 --- a/swh/scheduler/backend.py +++ b/swh/scheduler/backend.py @@ -1,1114 +1,1119 @@ # Copyright (C) 2015-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import datetime import json import logging from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from uuid import UUID import attr from psycopg2.errors import CardinalityViolation from psycopg2.extensions import AsIs import psycopg2.extras import psycopg2.pool from swh.core.db import BaseDb from swh.core.db.common import db_transaction from swh.scheduler.utils import utcnow from .exc import SchedulerException, StaleData, UnknownPolicy from .interface import ListedOriginPageToken, PaginatedListedOriginList from .model import ( LastVisitStatus, ListedOrigin, Lister, OriginVisitStats, SchedulerMetrics, ) logger = logging.getLogger(__name__) def adapt_LastVisitStatus(v: LastVisitStatus): return AsIs(f"'{v.value}'::last_visit_status") psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) psycopg2.extensions.register_adapter(LastVisitStatus, adapt_LastVisitStatus) psycopg2.extras.register_uuid() def format_query(query, keys): """Format a query with the given keys""" query_keys = ", ".join(keys) placeholders = ", ".join(["%s"] * len(keys)) return query.format(keys=query_keys, placeholders=placeholders) class SchedulerBackend: """Backend for the Software Heritage scheduling database. """ + current_version = 33 + def __init__(self, db, min_pool_conns=1, max_pool_conns=10): """ Args: db_conn: either a libpq connection string, or a psycopg2 connection """ if isinstance(db, psycopg2.extensions.connection): self._pool = None self._db = BaseDb(db) else: self._pool = psycopg2.pool.ThreadedConnectionPool( min_pool_conns, max_pool_conns, db, cursor_factory=psycopg2.extras.RealDictCursor, ) self._db = None + def get_current_version(self): + return self.current_version + def get_db(self): if self._db: return self._db return BaseDb.from_pool(self._pool) def put_db(self, db): if db is not self._db: db.put_conn() task_type_keys = [ "type", "description", "backend_name", "default_interval", "min_interval", "max_interval", "backoff_factor", "max_queue_length", "num_retries", "retry_delay", ] @db_transaction() def create_task_type(self, task_type, db=None, cur=None): """Create a new task type ready for scheduling. Args: task_type (dict): a dictionary with the following keys: - type (str): an identifier for the task type - description (str): a human-readable description of what the task does - backend_name (str): the name of the task in the job-scheduling backend - default_interval (datetime.timedelta): the default interval between two task runs - min_interval (datetime.timedelta): the minimum interval between two task runs - max_interval (datetime.timedelta): the maximum interval between two task runs - backoff_factor (float): the factor by which the interval changes at each run - max_queue_length (int): the maximum length of the task queue for this task type """ keys = [key for key in self.task_type_keys if key in task_type] query = format_query( """insert into task_type ({keys}) values ({placeholders}) on conflict do nothing""", keys, ) cur.execute(query, [task_type[key] for key in keys]) @db_transaction() def get_task_type(self, task_type_name, db=None, cur=None): """Retrieve the task type with id task_type_name""" query = format_query( "select {keys} from task_type where type=%s", self.task_type_keys, ) cur.execute(query, (task_type_name,)) return cur.fetchone() @db_transaction() def get_task_types(self, db=None, cur=None): """Retrieve all registered task types""" query = format_query("select {keys} from task_type", self.task_type_keys,) cur.execute(query) return cur.fetchall() @db_transaction() def get_listers(self, db=None, cur=None) -> List[Lister]: """Retrieve information about all listers from the database. """ select_cols = ", ".join(Lister.select_columns()) query = f""" select {select_cols} from listers """ cur.execute(query) return [Lister(**ret) for ret in cur.fetchall()] @db_transaction() def get_lister( self, name: str, instance_name: Optional[str] = None, db=None, cur=None ) -> Optional[Lister]: """Retrieve information about the given instance of the lister from the database. """ if instance_name is None: instance_name = "" select_cols = ", ".join(Lister.select_columns()) query = f""" select {select_cols} from listers where (name, instance_name) = (%s, %s) """ cur.execute(query, (name, instance_name)) ret = cur.fetchone() if not ret: return None return Lister(**ret) @db_transaction() def get_or_create_lister( self, name: str, instance_name: Optional[str] = None, db=None, cur=None ) -> Lister: """Retrieve information about the given instance of the lister from the database, or create the entry if it did not exist. """ if instance_name is None: instance_name = "" select_cols = ", ".join(Lister.select_columns()) insert_cols, insert_meta = ( ", ".join(tup) for tup in Lister.insert_columns_and_metavars() ) query = f""" with added as ( insert into listers ({insert_cols}) values ({insert_meta}) on conflict do nothing returning {select_cols} ) select {select_cols} from added union all select {select_cols} from listers where (name, instance_name) = (%(name)s, %(instance_name)s); """ cur.execute(query, attr.asdict(Lister(name=name, instance_name=instance_name))) return Lister(**cur.fetchone()) @db_transaction() def update_lister(self, lister: Lister, db=None, cur=None) -> Lister: """Update the state for the given lister instance in the database. Returns: a new Lister object, with all fields updated from the database Raises: StaleData if the `updated` timestamp for the lister instance in database doesn't match the one passed by the user. """ select_cols = ", ".join(Lister.select_columns()) set_vars = ", ".join( f"{col} = {meta}" for col, meta in zip(*Lister.insert_columns_and_metavars()) ) query = f"""update listers set {set_vars} where id=%(id)s and updated=%(updated)s returning {select_cols}""" cur.execute(query, attr.asdict(lister)) updated = cur.fetchone() if not updated: raise StaleData("Stale data; Lister state not updated") return Lister(**updated) @db_transaction() def record_listed_origins( self, listed_origins: Iterable[ListedOrigin], db=None, cur=None ) -> List[ListedOrigin]: """Record a set of origins that a lister has listed. This performs an "upsert": origins with the same (lister_id, url, visit_type) values are updated with new values for extra_loader_arguments, last_update and last_seen. """ pk_cols = ListedOrigin.primary_key_columns() select_cols = ListedOrigin.select_columns() insert_cols, insert_meta = ListedOrigin.insert_columns_and_metavars() deduplicated_origins = { tuple(getattr(origin, k) for k in pk_cols): origin for origin in listed_origins } upsert_cols = [col for col in insert_cols if col not in pk_cols] upsert_set = ", ".join(f"{col} = EXCLUDED.{col}" for col in upsert_cols) query = f"""INSERT into listed_origins ({", ".join(insert_cols)}) VALUES %s ON CONFLICT ({", ".join(pk_cols)}) DO UPDATE SET {upsert_set} RETURNING {", ".join(select_cols)} """ ret = psycopg2.extras.execute_values( cur=cur, sql=query, argslist=(attr.asdict(origin) for origin in deduplicated_origins.values()), template=f"({', '.join(insert_meta)})", page_size=1000, fetch=True, ) return [ListedOrigin(**d) for d in ret] @db_transaction() def get_listed_origins( self, lister_id: Optional[UUID] = None, url: Optional[str] = None, limit: int = 1000, page_token: Optional[ListedOriginPageToken] = None, db=None, cur=None, ) -> PaginatedListedOriginList: """Get information on the listed origins matching either the `url` or `lister_id`, or both arguments. """ query_filters: List[str] = [] query_params: List[Union[int, str, UUID, Tuple[UUID, str]]] = [] if lister_id: query_filters.append("lister_id = %s") query_params.append(lister_id) if url is not None: query_filters.append("url = %s") query_params.append(url) if page_token is not None: query_filters.append("(lister_id, url) > %s") # the typeshed annotation for tuple() is too strict. query_params.append(tuple(page_token)) # type: ignore query_params.append(limit) select_cols = ", ".join(ListedOrigin.select_columns()) if query_filters: where_clause = "where %s" % (" and ".join(query_filters)) else: where_clause = "" query = f"""SELECT {select_cols} from listed_origins {where_clause} ORDER BY lister_id, url LIMIT %s""" cur.execute(query, tuple(query_params)) origins = [ListedOrigin(**d) for d in cur] if len(origins) == limit: page_token = (str(origins[-1].lister_id), origins[-1].url) else: page_token = None return PaginatedListedOriginList(origins, page_token) @db_transaction() def grab_next_visits( self, visit_type: str, count: int, policy: str, enabled: bool = True, lister_uuid: Optional[str] = None, timestamp: Optional[datetime.datetime] = None, scheduled_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=7), failed_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=14), not_found_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=31), tablesample: Optional[float] = None, db=None, cur=None, ) -> List[ListedOrigin]: if timestamp is None: timestamp = utcnow() origin_select_cols = ", ".join(ListedOrigin.select_columns()) query_args: List[Any] = [] where_clauses = [] # list of (name, query) handled as CTEs before the main query common_table_expressions: List[Tuple[str, str]] = [] # "NOT enabled" = the lister said the origin no longer exists where_clauses.append("enabled" if enabled else "not enabled") # Only schedule visits of the given type where_clauses.append("visit_type = %s") query_args.append(visit_type) if scheduled_cooldown: # Don't re-schedule visits if they're already scheduled but we haven't # recorded a result yet, unless they've been scheduled more than a week # ago (it probably means we've lost them in flight somewhere). where_clauses.append( """origin_visit_stats.last_scheduled IS NULL OR origin_visit_stats.last_scheduled < GREATEST( %s, origin_visit_stats.last_visit ) """ ) query_args.append(timestamp - scheduled_cooldown) if failed_cooldown: # Don't retry failed origins too often where_clauses.append( "origin_visit_stats.last_visit_status is distinct from 'failed' " "or origin_visit_stats.last_visit < %s" ) query_args.append(timestamp - failed_cooldown) if not_found_cooldown: # Don't retry not found origins too often where_clauses.append( "origin_visit_stats.last_visit_status is distinct from 'not_found' " "or origin_visit_stats.last_visit < %s" ) query_args.append(timestamp - not_found_cooldown) if policy == "oldest_scheduled_first": order_by = "origin_visit_stats.last_scheduled NULLS FIRST" elif policy == "never_visited_oldest_update_first": # never visited origins have a NULL last_snapshot where_clauses.append("origin_visit_stats.last_snapshot IS NULL") # order by increasing last_update (oldest first) where_clauses.append("listed_origins.last_update IS NOT NULL") order_by = "listed_origins.last_update" elif policy == "already_visited_order_by_lag": # TODO: store "visit lag" in a materialized view? # visited origins have a NOT NULL last_snapshot where_clauses.append("origin_visit_stats.last_snapshot IS NOT NULL") # ignore origins we have visited after the known last update where_clauses.append("listed_origins.last_update IS NOT NULL") where_clauses.append( "listed_origins.last_update > origin_visit_stats.last_successful" ) # order by decreasing visit lag order_by = ( "listed_origins.last_update - origin_visit_stats.last_successful DESC" ) elif policy == "origins_without_last_update": where_clauses.append("last_update IS NULL") order_by = ", ".join( [ # By default, sort using the queue position. If the queue # position is null, then the origin has never been visited, # which we want to handle first "origin_visit_stats.next_visit_queue_position nulls first", # Schedule unknown origins in the order we've seen them "listed_origins.first_seen", ] ) # fmt: off # This policy requires updating the global queue position for this # visit type common_table_expressions.append(("update_queue_position", """ INSERT INTO visit_scheduler_queue_position(visit_type, position) SELECT visit_type, COALESCE(MAX(next_visit_queue_position), 0) FROM selected_origins GROUP BY visit_type ON CONFLICT(visit_type) DO UPDATE SET position=GREATEST( visit_scheduler_queue_position.position, EXCLUDED.position ) """)) # fmt: on else: raise UnknownPolicy(f"Unknown scheduling policy {policy}") if tablesample: table = "listed_origins tablesample SYSTEM (%s)" query_args.insert(0, tablesample) else: table = "listed_origins" if lister_uuid: where_clauses.append("lister_id = %s") query_args.append(lister_uuid) # fmt: off common_table_expressions.insert(0, ("selected_origins", f""" SELECT {origin_select_cols}, next_visit_queue_position FROM {table} LEFT JOIN origin_visit_stats USING (url, visit_type) WHERE ({") AND (".join(where_clauses)}) ORDER BY {order_by} LIMIT %s """)) # fmt: on query_args.append(count) # fmt: off common_table_expressions.append(("deduplicated_selected_origins", """ SELECT DISTINCT url, visit_type FROM selected_origins """)) # fmt: on # fmt: off common_table_expressions.append(("update_stats", """ INSERT INTO origin_visit_stats (url, visit_type, last_scheduled) SELECT url, visit_type, %s FROM deduplicated_selected_origins ON CONFLICT (url, visit_type) DO UPDATE SET last_scheduled = GREATEST( origin_visit_stats.last_scheduled, EXCLUDED.last_scheduled ) """)) # fmt: on query_args.append(timestamp) formatted_ctes = ",\n".join( f"{name} AS (\n{cte}\n)" for name, cte in common_table_expressions ) query = f""" WITH {formatted_ctes} SELECT {origin_select_cols} FROM selected_origins """ cur.execute(query, tuple(query_args)) return [ListedOrigin(**d) for d in cur] task_create_keys = [ "type", "arguments", "next_run", "policy", "status", "retries_left", "priority", ] task_keys = task_create_keys + ["id", "current_interval"] @db_transaction() def create_tasks(self, tasks, policy="recurring", db=None, cur=None): """Create new tasks. Args: tasks (list): each task is a dictionary with the following keys: - type (str): the task type - arguments (dict): the arguments for the task runner, keys: - args (list of str): arguments - kwargs (dict str -> str): keyword arguments - next_run (datetime.datetime): the next scheduled run for the task Returns: a list of created tasks. """ cur.execute("select swh_scheduler_mktemp_task()") db.copy_to( tasks, "tmp_task", self.task_create_keys, default_values={"policy": policy, "status": "next_run_not_scheduled"}, cur=cur, ) query = format_query( "select {keys} from swh_scheduler_create_tasks_from_temp()", self.task_keys, ) cur.execute(query) return cur.fetchall() @db_transaction() def set_status_tasks( self, task_ids: List[int], status: str = "disabled", next_run: Optional[datetime.datetime] = None, db=None, cur=None, ): """Set the tasks' status whose ids are listed. If given, also set the next_run date. """ if not task_ids: return query = ["UPDATE task SET status = %s"] args: List[Any] = [status] if next_run: query.append(", next_run = %s") args.append(next_run) query.append(" WHERE id IN %s") args.append(tuple(task_ids)) cur.execute("".join(query), args) @db_transaction() def disable_tasks(self, task_ids, db=None, cur=None): """Disable the tasks whose ids are listed.""" return self.set_status_tasks(task_ids, db=db, cur=cur) @db_transaction() def search_tasks( self, task_id=None, task_type=None, status=None, priority=None, policy=None, before=None, after=None, limit=None, db=None, cur=None, ): """Search tasks from selected criterions""" where = [] args = [] if task_id: if isinstance(task_id, (str, int)): where.append("id = %s") else: where.append("id in %s") task_id = tuple(task_id) args.append(task_id) if task_type: if isinstance(task_type, str): where.append("type = %s") else: where.append("type in %s") task_type = tuple(task_type) args.append(task_type) if status: if isinstance(status, str): where.append("status = %s") else: where.append("status in %s") status = tuple(status) args.append(status) if priority: if isinstance(priority, str): where.append("priority = %s") else: priority = tuple(priority) where.append("priority in %s") args.append(priority) if policy: where.append("policy = %s") args.append(policy) if before: where.append("next_run <= %s") args.append(before) if after: where.append("next_run >= %s") args.append(after) query = "select * from task" if where: query += " where " + " and ".join(where) if limit: query += " limit %s :: bigint" args.append(limit) cur.execute(query, args) return cur.fetchall() @db_transaction() def get_tasks(self, task_ids, db=None, cur=None): """Retrieve the info of tasks whose ids are listed.""" query = format_query("select {keys} from task where id in %s", self.task_keys) cur.execute(query, (tuple(task_ids),)) return cur.fetchall() @db_transaction() def peek_ready_tasks( self, task_type: str, timestamp: Optional[datetime.datetime] = None, num_tasks: Optional[int] = None, db=None, cur=None, ) -> List[Dict]: if timestamp is None: timestamp = utcnow() cur.execute( """select * from swh_scheduler_peek_no_priority_tasks( %s, %s, %s :: bigint)""", (task_type, timestamp, num_tasks), ) logger.debug("PEEK %s => %s" % (task_type, cur.rowcount)) return cur.fetchall() @db_transaction() def grab_ready_tasks( self, task_type: str, timestamp: Optional[datetime.datetime] = None, num_tasks: Optional[int] = None, db=None, cur=None, ) -> List[Dict]: if timestamp is None: timestamp = utcnow() cur.execute( """select * from swh_scheduler_grab_ready_tasks( %s, %s, %s :: bigint)""", (task_type, timestamp, num_tasks), ) logger.debug("GRAB %s => %s" % (task_type, cur.rowcount)) return cur.fetchall() @db_transaction() def peek_ready_priority_tasks( self, task_type: str, timestamp: Optional[datetime.datetime] = None, num_tasks: Optional[int] = None, db=None, cur=None, ) -> List[Dict]: if timestamp is None: timestamp = utcnow() cur.execute( """select * from swh_scheduler_peek_any_ready_priority_tasks( %s, %s, %s :: bigint)""", (task_type, timestamp, num_tasks), ) logger.debug("PEEK %s => %s", task_type, cur.rowcount) return cur.fetchall() @db_transaction() def grab_ready_priority_tasks( self, task_type: str, timestamp: Optional[datetime.datetime] = None, num_tasks: Optional[int] = None, db=None, cur=None, ) -> List[Dict]: if timestamp is None: timestamp = utcnow() cur.execute( """select * from swh_scheduler_grab_any_ready_priority_tasks( %s, %s, %s :: bigint)""", (task_type, timestamp, num_tasks), ) logger.debug("GRAB %s => %s", task_type, cur.rowcount) return cur.fetchall() task_run_create_keys = ["task", "backend_id", "scheduled", "metadata"] @db_transaction() def schedule_task_run( self, task_id, backend_id, metadata=None, timestamp=None, db=None, cur=None ): """Mark a given task as scheduled, adding a task_run entry in the database. Args: task_id (int): the identifier for the task being scheduled backend_id (str): the identifier of the job in the backend metadata (dict): metadata to add to the task_run entry timestamp (datetime.datetime): the instant the event occurred Returns: a fresh task_run entry """ if metadata is None: metadata = {} if timestamp is None: timestamp = utcnow() cur.execute( "select * from swh_scheduler_schedule_task_run(%s, %s, %s, %s)", (task_id, backend_id, metadata, timestamp), ) return cur.fetchone() @db_transaction() def mass_schedule_task_runs(self, task_runs, db=None, cur=None): """Schedule a bunch of task runs. Args: task_runs (list): a list of dicts with keys: - task (int): the identifier for the task being scheduled - backend_id (str): the identifier of the job in the backend - metadata (dict): metadata to add to the task_run entry - scheduled (datetime.datetime): the instant the event occurred Returns: None """ cur.execute("select swh_scheduler_mktemp_task_run()") db.copy_to(task_runs, "tmp_task_run", self.task_run_create_keys, cur=cur) cur.execute("select swh_scheduler_schedule_task_run_from_temp()") @db_transaction() def start_task_run( self, backend_id, metadata=None, timestamp=None, db=None, cur=None ): """Mark a given task as started, updating the corresponding task_run entry in the database. Args: backend_id (str): the identifier of the job in the backend metadata (dict): metadata to add to the task_run entry timestamp (datetime.datetime): the instant the event occurred Returns: the updated task_run entry """ if metadata is None: metadata = {} if timestamp is None: timestamp = utcnow() cur.execute( "select * from swh_scheduler_start_task_run(%s, %s, %s)", (backend_id, metadata, timestamp), ) return cur.fetchone() @db_transaction() def end_task_run( self, backend_id, status, metadata=None, timestamp=None, result=None, db=None, cur=None, ): """Mark a given task as ended, updating the corresponding task_run entry in the database. Args: backend_id (str): the identifier of the job in the backend status (str): how the task ended; one of: 'eventful', 'uneventful', 'failed' metadata (dict): metadata to add to the task_run entry timestamp (datetime.datetime): the instant the event occurred Returns: the updated task_run entry """ if metadata is None: metadata = {} if timestamp is None: timestamp = utcnow() cur.execute( "select * from swh_scheduler_end_task_run(%s, %s, %s, %s)", (backend_id, status, metadata, timestamp), ) return cur.fetchone() @db_transaction() def filter_task_to_archive( self, after_ts: str, before_ts: str, limit: int = 10, page_token: Optional[str] = None, db=None, cur=None, ) -> Dict[str, Any]: """Compute the tasks to archive within the datetime interval [after_ts, before_ts[. The method returns a paginated result. Returns: dict with the following keys: - **next_page_token**: opaque token to be used as `page_token` to retrieve the next page of result. If absent, there is no more pages to gather. - **tasks**: list of task dictionaries with the following keys: **id** (str): origin task id **started** (Optional[datetime]): started date **scheduled** (datetime): scheduled date **arguments** (json dict): task's arguments ... """ assert not page_token or isinstance(page_token, str) last_id = -1 if page_token is None else int(page_token) tasks = [] cur.execute( "select * from swh_scheduler_task_to_archive(%s, %s, %s, %s)", (after_ts, before_ts, last_id, limit + 1), ) for row in cur: task = dict(row) # nested type index does not accept bare values # transform it as a dict to comply with this task["arguments"]["args"] = { i: v for i, v in enumerate(task["arguments"]["args"]) } kwargs = task["arguments"]["kwargs"] task["arguments"]["kwargs"] = json.dumps(kwargs) tasks.append(task) if len(tasks) >= limit + 1: # remains data, add pagination information result = { "tasks": tasks[:limit], "next_page_token": str(tasks[-1]["task_id"]), } else: result = {"tasks": tasks} return result @db_transaction() def delete_archived_tasks(self, task_ids, db=None, cur=None): """Delete archived tasks as much as possible. Only the task_ids whose complete associated task_run have been cleaned up will be. """ _task_ids = _task_run_ids = [] for task_id in task_ids: _task_ids.append(task_id["task_id"]) _task_run_ids.append(task_id["task_run_id"]) cur.execute( "select * from swh_scheduler_delete_archived_tasks(%s, %s)", (_task_ids, _task_run_ids), ) task_run_keys = [ "id", "task", "backend_id", "scheduled", "started", "ended", "metadata", "status", ] @db_transaction() def get_task_runs(self, task_ids, limit=None, db=None, cur=None): """Search task run for a task id""" where = [] args = [] if task_ids: if isinstance(task_ids, (str, int)): where.append("task = %s") else: where.append("task in %s") task_ids = tuple(task_ids) args.append(task_ids) else: return () query = "select * from task_run where " + " and ".join(where) if limit: query += " limit %s :: bigint" args.append(limit) cur.execute(query, args) return cur.fetchall() @db_transaction() def origin_visit_stats_upsert( self, origin_visit_stats: Iterable[OriginVisitStats], db=None, cur=None ) -> None: pk_cols = OriginVisitStats.primary_key_columns() insert_cols, insert_meta = OriginVisitStats.insert_columns_and_metavars() upsert_cols = [col for col in insert_cols if col not in pk_cols] upsert_set = ", ".join( f"{col} = coalesce(EXCLUDED.{col}, ovi.{col})" for col in upsert_cols ) query = f""" INSERT into origin_visit_stats AS ovi ({", ".join(insert_cols)}) VALUES %s ON CONFLICT ({", ".join(pk_cols)}) DO UPDATE SET {upsert_set} """ try: psycopg2.extras.execute_values( cur=cur, sql=query, argslist=( attr.asdict(visit_stats) for visit_stats in origin_visit_stats ), template=f"({', '.join(insert_meta)})", page_size=1000, fetch=False, ) except CardinalityViolation as e: raise SchedulerException(repr(e)) @db_transaction() def origin_visit_stats_get( self, ids: Iterable[Tuple[str, str]], db=None, cur=None ) -> List[OriginVisitStats]: if not ids: return [] primary_keys = tuple((origin, visit_type) for (origin, visit_type) in ids) query = format_query( """ SELECT {keys} FROM (VALUES %s) as stats(url, visit_type) INNER JOIN origin_visit_stats USING (url, visit_type) """, OriginVisitStats.select_columns(), ) rows = psycopg2.extras.execute_values( cur=cur, sql=query, argslist=primary_keys, fetch=True ) return [OriginVisitStats(**row) for row in rows] @db_transaction() def visit_scheduler_queue_position_get(self, db=None, cur=None) -> Dict[str, int]: cur.execute("SELECT visit_type, position FROM visit_scheduler_queue_position") return {row["visit_type"]: row["position"] for row in cur} @db_transaction() def visit_scheduler_queue_position_set( self, visit_type: str, position: int, db=None, cur=None, ) -> None: query = """ INSERT INTO visit_scheduler_queue_position(visit_type, position) VALUES(%s, %s) ON CONFLICT(visit_type) DO UPDATE SET position=EXCLUDED.position """ cur.execute(query, (visit_type, position)) @db_transaction() def update_metrics( self, lister_id: Optional[UUID] = None, timestamp: Optional[datetime.datetime] = None, db=None, cur=None, ) -> List[SchedulerMetrics]: """Update the performance metrics of this scheduler instance. Returns the updated metrics. Args: lister_id: if passed, update the metrics only for this lister instance timestamp: if passed, the date at which we're updating the metrics, defaults to the database NOW() """ query = format_query( "SELECT {keys} FROM update_metrics(%s, %s)", SchedulerMetrics.select_columns(), ) cur.execute(query, (lister_id, timestamp)) return [SchedulerMetrics(**row) for row in cur.fetchall()] @db_transaction() def get_metrics( self, lister_id: Optional[UUID] = None, visit_type: Optional[str] = None, db=None, cur=None, ) -> List[SchedulerMetrics]: """Retrieve the performance metrics of this scheduler instance. Args: lister_id: filter the metrics for this lister instance only visit_type: filter the metrics for this visit type only """ where_filters = [] where_args = [] if lister_id: where_filters.append("lister_id = %s") where_args.append(str(lister_id)) if visit_type: where_filters.append("visit_type = %s") where_args.append(visit_type) where_clause = "" if where_filters: where_clause = f"where {' and '.join(where_filters)}" query = format_query( "SELECT {keys} FROM scheduler_metrics %s" % where_clause, SchedulerMetrics.select_columns(), ) cur.execute(query, tuple(where_args)) return [SchedulerMetrics(**row) for row in cur.fetchall()] diff --git a/swh/scheduler/pytest_plugin.py b/swh/scheduler/pytest_plugin.py index 297d4b4..870d9e2 100644 --- a/swh/scheduler/pytest_plugin.py +++ b/swh/scheduler/pytest_plugin.py @@ -1,105 +1,111 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import timedelta -import os +from functools import partial from celery.contrib.testing import worker from celery.contrib.testing.app import TestApp, setup_default_app import pkg_resources import pytest +from pytest_postgresql import factories -from swh.core.db.pytest_plugin import postgresql_fact -import swh.scheduler +from swh.core.db.pytest_plugin import initialize_database_for_module, postgresql_fact from swh.scheduler import get_scheduler - -SQL_DIR = os.path.join(os.path.dirname(swh.scheduler.__file__), "sql") +from swh.scheduler.backend import SchedulerBackend # celery tasks for testing purpose; tasks themselves should be # in swh/scheduler/tests/tasks.py TASK_NAMES = ["ping", "multiping", "add", "error", "echo"] -postgresql_scheduler = postgresql_fact( - "postgresql_proc", +scheduler_postgresql_proc = factories.postgresql_proc( dbname="scheduler", - dump_files=os.path.join(SQL_DIR, "*.sql"), - no_truncate_tables={"dbversion", "priority_ratio"}, + load=[ + partial( + initialize_database_for_module, + modname="scheduler", + version=SchedulerBackend.current_version, + ) + ], ) +postgresql_scheduler = postgresql_fact("scheduler_postgresql_proc") + + @pytest.fixture def swh_scheduler_config(request, postgresql_scheduler): return { "db": postgresql_scheduler.dsn, } @pytest.fixture def swh_scheduler(swh_scheduler_config): scheduler = get_scheduler("local", **swh_scheduler_config) for taskname in TASK_NAMES: scheduler.create_task_type( { "type": "swh-test-{}".format(taskname), "description": "The {} testing task".format(taskname), "backend_name": "swh.scheduler.tests.tasks.{}".format(taskname), "default_interval": timedelta(days=1), "min_interval": timedelta(hours=6), "max_interval": timedelta(days=12), } ) return scheduler # this alias is used to be able to easily instantiate a db-backed Scheduler # eg. for the RPC client/server test suite. swh_db_scheduler = swh_scheduler @pytest.fixture(scope="session") def swh_scheduler_celery_app(): """Set up a Celery app as swh.scheduler and swh worker tests would expect it""" test_app = TestApp( set_as_current=True, enable_logging=True, task_cls="swh.scheduler.task:SWHTask", config={ "accept_content": ["application/x-msgpack", "application/json"], "broker_url": "memory://guest@localhost//", "task_serializer": "msgpack", "result_serializer": "json", }, ) with setup_default_app(test_app, use_trap=False): from swh.scheduler.celery_backend import config config.app = test_app test_app.set_default() test_app.set_current() yield test_app @pytest.fixture(scope="session") def swh_scheduler_celery_includes(): """List of task modules that should be loaded by the swh_scheduler_celery_worker on startup.""" task_modules = ["swh.scheduler.tests.tasks"] for entrypoint in pkg_resources.iter_entry_points("swh.workers"): task_modules.extend(entrypoint.load()().get("task_modules", [])) return task_modules @pytest.fixture(scope="session") def swh_scheduler_celery_worker( swh_scheduler_celery_app, swh_scheduler_celery_includes, ): """Spawn a worker""" for module in swh_scheduler_celery_includes: swh_scheduler_celery_app.loader.import_task_module(module) with worker.start_worker(swh_scheduler_celery_app, pool="solo") as w: yield w diff --git a/swh/scheduler/simulator/origins.py b/swh/scheduler/simulator/origins.py index a0ec081..546bfc3 100644 --- a/swh/scheduler/simulator/origins.py +++ b/swh/scheduler/simulator/origins.py @@ -1,227 +1,227 @@ # Copyright (C) 2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """This module implements a model of the frequency of updates of an origin and how long it takes to load it. For each origin, a commit frequency is chosen deterministically based on the hash of its URL and assume all origins were created on an arbitrary epoch. From this we compute a number of commits, that is the product of these two. And the run time of a load task is approximated as proportional to the number of commits since the previous visit of the origin (possibly 0).""" from datetime import datetime, timedelta, timezone import hashlib import logging from typing import Dict, Generator, Iterator, List, Optional, Tuple import uuid import attr from simpy import Event from swh.model.model import OriginVisitStatus from swh.scheduler.model import ListedOrigin from .common import Environment, Queue, Task, TaskEvent logger = logging.getLogger(__name__) _nb_generated_origins = 0 _visit_times: Dict[Tuple[str, str], datetime] = {} """Cache of the time of the last visit of (visit_type, origin_url), to spare an SQL query (high latency).""" def generate_listed_origin( lister_id: uuid.UUID, now: Optional[datetime] = None ) -> ListedOrigin: """Returns a globally unique new origin. Seed the `last_update` value according to the OriginModel and the passed timestamp. Arguments: lister: instance of the lister that generated this origin now: time of listing, to emulate last_update (defaults to :func:`datetime.now`) """ global _nb_generated_origins _nb_generated_origins += 1 assert _nb_generated_origins < 10 ** 6, "Too many origins!" if now is None: now = datetime.now(tz=timezone.utc) url = f"https://example.com/{_nb_generated_origins:06d}.git" - visit_type = "git" + visit_type = "test-git" origin = OriginModel(visit_type, url) return ListedOrigin( lister_id=lister_id, url=url, visit_type=visit_type, last_update=origin.get_last_update(now), ) class OriginModel: MIN_RUN_TIME = 0.5 """Minimal run time for a visit (retrieved from production data)""" MAX_RUN_TIME = 7200 """Max run time for a visit""" PER_COMMIT_RUN_TIME = 0.1 """Run time per commit""" EPOCH = datetime(2015, 9, 1, 0, 0, 0, tzinfo=timezone.utc) """The origin of all origins (at least according to Software Heritage)""" def __init__(self, type: str, origin: str): self.type = type self.origin = origin def seconds_between_commits(self): """Returns a random 'average time between two commits' of this origin, used to estimate the run time of a load task, and how much the loading architecture is lagging behind origin updates.""" n_bytes = 2 num_buckets = 2 ** (8 * n_bytes) # Deterministic seed to generate "random" characteristics of this origin bucket = int.from_bytes( hashlib.md5(self.origin.encode()).digest()[0:n_bytes], "little" ) # minimum: 1 second (bucket == 0) # max: 10 years (bucket == num_buckets - 1) ten_y = 10 * 365 * 24 * 3600 return ten_y ** (bucket / num_buckets) # return 1 + (ten_y - 1) * (bucket / (num_buckets - 1)) def get_last_update(self, now: datetime) -> datetime: """Get the last_update value for this origin. We assume that the origin had its first commit at `EPOCH`, and that one commit happened every `self.seconds_between_commits()`. This returns the last commit date before or equal to `now`. """ _, time_since_last_commit = divmod( (now - self.EPOCH).total_seconds(), self.seconds_between_commits() ) return now - timedelta(seconds=time_since_last_commit) def get_current_snapshot_id(self, now: datetime) -> bytes: """Get the current snapshot for this origin. To generate a snapshot id, we calculate the number of commits since the EPOCH, and hash it alongside the origin type and url. """ commits_since_epoch, _ = divmod( (now - self.EPOCH).total_seconds(), self.seconds_between_commits() ) return hashlib.sha1( f"{self.type} {self.origin} {commits_since_epoch}".encode() ).digest() def load_task_characteristics( self, now: datetime ) -> Tuple[float, str, Optional[bytes]]: """Returns the (run_time, end_status, snapshot id) of the next origin visit.""" current_snapshot = self.get_current_snapshot_id(now) key = (self.type, self.origin) last_visit = _visit_times.get(key, now - timedelta(days=365)) time_since_last_successful_run = now - last_visit _visit_times[key] = now seconds_between_commits = self.seconds_between_commits() seconds_since_last_successful = time_since_last_successful_run.total_seconds() n_commits = int(seconds_since_last_successful / seconds_between_commits) logger.debug( "%s characteristics %s origin=%s: Interval: %s, n_commits: %s", now, self.type, self.origin, timedelta(seconds=seconds_between_commits), n_commits, ) run_time = self.MIN_RUN_TIME + self.PER_COMMIT_RUN_TIME * n_commits if run_time > self.MAX_RUN_TIME: # Long visits usually fail return (self.MAX_RUN_TIME, "partial", None) else: return (run_time, "full", current_snapshot) def lister_process( env: Environment, lister_id: uuid.UUID ) -> Generator[Event, Event, None]: """Every hour, generate new origins and update the `last_update` field for the ones this process generated in the past""" NUM_NEW_ORIGINS = 100 origins: List[ListedOrigin] = [] while True: updated_origins = [] for origin in origins: model = OriginModel(origin.visit_type, origin.url) updated_origins.append( attr.evolve(origin, last_update=model.get_last_update(env.time)) ) origins = updated_origins origins.extend( generate_listed_origin(lister_id, now=env.time) for _ in range(NUM_NEW_ORIGINS) ) env.scheduler.record_listed_origins(origins) yield env.timeout(3600) def load_task_process( env: Environment, task: Task, status_queue: Queue ) -> Iterator[Event]: """A loading task. This pushes OriginVisitStatus objects to the status_queue to simulate the visible outcomes of the task. Uses the `load_task_duration` function to determine its run time. """ status = OriginVisitStatus( origin=task.origin, visit=42, type=task.visit_type, status="created", date=env.time, snapshot=None, ) logger.debug("%s task %s origin=%s: Start", env.time, task.visit_type, task.origin) yield status_queue.put(TaskEvent(task=task, status=status)) origin_model = OriginModel(task.visit_type, task.origin) (run_time, end_status, snapshot) = origin_model.load_task_characteristics(env.time) yield env.timeout(run_time) logger.debug("%s task %s origin=%s: End", env.time, task.visit_type, task.origin) yield status_queue.put( TaskEvent( task=task, status=attr.evolve( status, status=end_status, date=env.time, snapshot=snapshot ), ) ) env.report.record_visit( (task.visit_type, task.origin), run_time, end_status, snapshot ) diff --git a/swh/scheduler/sql/30-schema.sql b/swh/scheduler/sql/30-schema.sql index 7de6d48..bec9727 100644 --- a/swh/scheduler/sql/30-schema.sql +++ b/swh/scheduler/sql/30-schema.sql @@ -1,229 +1,214 @@ -create table dbversion -( - version int primary key, - release timestamptz not null, - description text not null -); - -comment on table dbversion is 'Schema update tracking'; -comment on column dbversion.version is 'SQL schema version'; -comment on column dbversion.release is 'Version deployment timestamp'; -comment on column dbversion.description is 'Version description'; - -insert into dbversion (version, release, description) - values (32, now(), 'Work In Progress'); - create table task_type ( type text primary key, description text not null, backend_name text not null, default_interval interval, min_interval interval, max_interval interval, backoff_factor float, max_queue_length bigint, num_retries bigint, retry_delay interval ); comment on table task_type is 'Types of schedulable tasks'; comment on column task_type.type is 'Short identifier for the task type'; comment on column task_type.description is 'Human-readable task description'; comment on column task_type.backend_name is 'Name of the task in the job-running backend'; comment on column task_type.default_interval is 'Default interval for newly scheduled tasks'; comment on column task_type.min_interval is 'Minimum interval between two runs of a task'; comment on column task_type.max_interval is 'Maximum interval between two runs of a task'; comment on column task_type.backoff_factor is 'Adjustment factor for the backoff between two task runs'; comment on column task_type.max_queue_length is 'Maximum length of the queue for this type of tasks'; comment on column task_type.num_retries is 'Default number of retries on transient failures'; comment on column task_type.retry_delay is 'Retry delay for the task'; create type task_status as enum ('next_run_not_scheduled', 'next_run_scheduled', 'completed', 'disabled'); comment on type task_status is 'Status of a given task'; create type task_policy as enum ('recurring', 'oneshot'); comment on type task_policy is 'Recurrence policy of the given task'; create type task_priority as enum('high', 'normal', 'low'); comment on type task_priority is 'Priority of the given task'; create table priority_ratio( id task_priority primary key, ratio float not null ); comment on table priority_ratio is 'Oneshot task''s reading ratio per priority'; comment on column priority_ratio.id is 'Task priority id'; comment on column priority_ratio.ratio is 'Percentage of tasks to read per priority'; insert into priority_ratio (id, ratio) values ('high', 0.5); insert into priority_ratio (id, ratio) values ('normal', 0.3); insert into priority_ratio (id, ratio) values ('low', 0.2); create table task ( id bigserial primary key, type text not null references task_type(type), arguments jsonb not null, next_run timestamptz not null, current_interval interval, status task_status not null, policy task_policy not null default 'recurring', retries_left bigint not null default 0, priority task_priority references priority_ratio(id), check (policy <> 'recurring' or current_interval is not null) ); comment on table task is 'Schedule of recurring tasks'; comment on column task.arguments is 'Arguments passed to the underlying job scheduler. ' 'Contains two keys, ''args'' (list) and ''kwargs'' (object).'; comment on column task.next_run is 'The next run of this task should be run on or after that time'; comment on column task.current_interval is 'The interval between two runs of this task, ' 'taking into account the backoff factor'; comment on column task.policy is 'Whether the task is one-shot or recurring'; comment on column task.retries_left is 'The number of "short delay" retries of the task in case of ' 'transient failure'; comment on column task.priority is 'Policy of the given task'; comment on column task.id is 'Task Identifier'; comment on column task.type is 'References task_type table'; comment on column task.status is 'Task status (''next_run_not_scheduled'', ''next_run_scheduled'', ''completed'', ''disabled'')'; create type task_run_status as enum ('scheduled', 'started', 'eventful', 'uneventful', 'failed', 'permfailed', 'lost'); comment on type task_run_status is 'Status of a given task run'; create table task_run ( id bigserial primary key, task bigint not null references task(id), backend_id text, scheduled timestamptz, started timestamptz, ended timestamptz, metadata jsonb, status task_run_status not null default 'scheduled' ); comment on table task_run is 'History of task runs sent to the job-running backend'; comment on column task_run.backend_id is 'id of the task run in the job-running backend'; comment on column task_run.metadata is 'Useful metadata for the given task run. ' 'For instance, the worker that took on the job, ' 'or the logs for the run.'; comment on column task_run.id is 'Task run identifier'; comment on column task_run.task is 'References task table'; comment on column task_run.scheduled is 'Scheduled run time for task'; comment on column task_run.started is 'Task starting time'; comment on column task_run.ended is 'Task ending time'; create table if not exists listers ( id uuid primary key default uuid_generate_v4(), name text not null, instance_name text not null, created timestamptz not null default now(), -- auto_now_add in the model current_state jsonb not null, updated timestamptz not null ); comment on table listers is 'Lister instances known to the origin visit scheduler'; comment on column listers.name is 'Name of the lister (e.g. github, gitlab, debian, ...)'; comment on column listers.instance_name is 'Name of the current instance of this lister (e.g. framagit, bitbucket, ...)'; comment on column listers.created is 'Timestamp at which the lister was originally created'; comment on column listers.current_state is 'Known current state of this lister'; comment on column listers.updated is 'Timestamp at which the lister state was last updated'; create table if not exists listed_origins ( -- Basic information lister_id uuid not null references listers(id), url text not null, visit_type text not null, extra_loader_arguments jsonb not null, -- Whether this origin still exists or not enabled boolean not null, -- time-based information first_seen timestamptz not null default now(), last_seen timestamptz not null, -- potentially provided by the lister last_update timestamptz, primary key (lister_id, url, visit_type) ); comment on table listed_origins is 'Origins known to the origin visit scheduler'; comment on column listed_origins.lister_id is 'Lister instance which owns this origin'; comment on column listed_origins.url is 'URL of the origin listed'; comment on column listed_origins.visit_type is 'Type of the visit which should be scheduled for the given url'; comment on column listed_origins.extra_loader_arguments is 'Extra arguments that should be passed to the loader for this origin'; comment on column listed_origins.enabled is 'Whether this origin has been seen during the last listing, and visits should be scheduled.'; comment on column listed_origins.first_seen is 'Time at which the origin was first seen by a lister'; comment on column listed_origins.last_seen is 'Time at which the origin was last seen by the lister'; comment on column listed_origins.last_update is 'Time of the last update to the origin recorded by the remote'; create type last_visit_status as enum ('successful', 'failed', 'not_found'); comment on type last_visit_status is 'Record of the status of the last visit of an origin'; create table origin_visit_stats ( url text not null, visit_type text not null, last_successful timestamptz, last_visit timestamptz, last_visit_status last_visit_status, -- visit scheduling information last_scheduled timestamptz, -- last snapshot resulting from an eventful visit last_snapshot bytea, -- position in the global queue, at which time we expect the origin to have new -- objects next_visit_queue_position bigint, -- duration that we expect to wait between visits of this origin next_position_offset int not null default 4, successive_visits int not null default 1, primary key (url, visit_type) ); comment on table origin_visit_stats is 'Aggregated information on visits for each origin in the archive'; comment on column origin_visit_stats.url is 'Origin URL'; comment on column origin_visit_stats.visit_type is 'Type of the visit for the given url'; comment on column origin_visit_stats.last_successful is 'Date of the last successful visit, at which we recorded the `last_snapshot`'; comment on column origin_visit_stats.last_visit is 'Date of the last visit overall'; comment on column origin_visit_stats.last_visit_status is 'Status of the last visit'; comment on column origin_visit_stats.last_scheduled is 'Time when this origin was scheduled to be visited last'; comment on column origin_visit_stats.last_snapshot is 'sha1_git of the last visit snapshot'; comment on column origin_visit_stats.next_visit_queue_position is 'Position in the global per origin-type queue at which some new objects are expected to be found'; comment on column origin_visit_stats.next_position_offset is 'Duration that we expect to wait between visits of this origin'; comment on column origin_visit_stats.successive_visits is 'number of successive visits with the same status'; create table visit_scheduler_queue_position ( visit_type text not null, position bigint not null, primary key (visit_type) ); comment on table visit_scheduler_queue_position is 'Current queue position for the recurrent visit scheduler'; comment on column visit_scheduler_queue_position.visit_type is 'Visit type'; comment on column visit_scheduler_queue_position.position is 'Current position for the runner of this visit type'; create table scheduler_metrics ( lister_id uuid not null references listers(id), visit_type text not null, last_update timestamptz not null, origins_known int not null default 0, origins_enabled int not null default 0, origins_never_visited int not null default 0, origins_with_pending_changes int not null default 0, primary key (lister_id, visit_type) ); comment on table scheduler_metrics is 'Cache of per-lister metrics for the scheduler, collated between the listed_origins and origin_visit_stats tables.'; comment on column scheduler_metrics.lister_id is 'Lister instance on which metrics have been aggregated'; comment on column scheduler_metrics.visit_type is 'Visit type on which metrics have been aggregated'; comment on column scheduler_metrics.last_update is 'Last update of these metrics'; comment on column scheduler_metrics.origins_known is 'Number of known (enabled or disabled) origins'; comment on column scheduler_metrics.origins_enabled is 'Number of origins that were present in the latest listing'; comment on column scheduler_metrics.origins_never_visited is 'Number of origins that have never been successfully visited'; comment on column scheduler_metrics.origins_with_pending_changes is 'Number of enabled origins with known activity since our last visit'; diff --git a/sql/updates/02.sql b/swh/scheduler/sql/upgrades/02.sql similarity index 100% rename from sql/updates/02.sql rename to swh/scheduler/sql/upgrades/02.sql diff --git a/sql/updates/03.sql b/swh/scheduler/sql/upgrades/03.sql similarity index 100% rename from sql/updates/03.sql rename to swh/scheduler/sql/upgrades/03.sql diff --git a/sql/updates/04.sql b/swh/scheduler/sql/upgrades/04.sql similarity index 100% rename from sql/updates/04.sql rename to swh/scheduler/sql/upgrades/04.sql diff --git a/sql/updates/05.sql b/swh/scheduler/sql/upgrades/05.sql similarity index 100% rename from sql/updates/05.sql rename to swh/scheduler/sql/upgrades/05.sql diff --git a/sql/updates/06.sql b/swh/scheduler/sql/upgrades/06.sql similarity index 100% rename from sql/updates/06.sql rename to swh/scheduler/sql/upgrades/06.sql diff --git a/sql/updates/07.sql b/swh/scheduler/sql/upgrades/07.sql similarity index 100% rename from sql/updates/07.sql rename to swh/scheduler/sql/upgrades/07.sql diff --git a/sql/updates/08.sql b/swh/scheduler/sql/upgrades/08.sql similarity index 100% rename from sql/updates/08.sql rename to swh/scheduler/sql/upgrades/08.sql diff --git a/sql/updates/09.sql b/swh/scheduler/sql/upgrades/09.sql similarity index 100% rename from sql/updates/09.sql rename to swh/scheduler/sql/upgrades/09.sql diff --git a/sql/updates/10.sql b/swh/scheduler/sql/upgrades/10.sql similarity index 100% rename from sql/updates/10.sql rename to swh/scheduler/sql/upgrades/10.sql diff --git a/sql/updates/11.sql b/swh/scheduler/sql/upgrades/11.sql similarity index 100% rename from sql/updates/11.sql rename to swh/scheduler/sql/upgrades/11.sql diff --git a/sql/updates/12.sql b/swh/scheduler/sql/upgrades/12.sql similarity index 100% rename from sql/updates/12.sql rename to swh/scheduler/sql/upgrades/12.sql diff --git a/sql/updates/13.sql b/swh/scheduler/sql/upgrades/13.sql similarity index 100% rename from sql/updates/13.sql rename to swh/scheduler/sql/upgrades/13.sql diff --git a/sql/updates/14.sql b/swh/scheduler/sql/upgrades/14.sql similarity index 100% rename from sql/updates/14.sql rename to swh/scheduler/sql/upgrades/14.sql diff --git a/sql/updates/15.sql b/swh/scheduler/sql/upgrades/15.sql similarity index 100% rename from sql/updates/15.sql rename to swh/scheduler/sql/upgrades/15.sql diff --git a/sql/updates/16.sql b/swh/scheduler/sql/upgrades/16.sql similarity index 100% rename from sql/updates/16.sql rename to swh/scheduler/sql/upgrades/16.sql diff --git a/sql/updates/17.sql b/swh/scheduler/sql/upgrades/17.sql similarity index 100% rename from sql/updates/17.sql rename to swh/scheduler/sql/upgrades/17.sql diff --git a/sql/updates/18.sql b/swh/scheduler/sql/upgrades/18.sql similarity index 100% rename from sql/updates/18.sql rename to swh/scheduler/sql/upgrades/18.sql diff --git a/sql/updates/19.sql b/swh/scheduler/sql/upgrades/19.sql similarity index 100% rename from sql/updates/19.sql rename to swh/scheduler/sql/upgrades/19.sql diff --git a/sql/updates/20.sql b/swh/scheduler/sql/upgrades/20.sql similarity index 100% rename from sql/updates/20.sql rename to swh/scheduler/sql/upgrades/20.sql diff --git a/sql/updates/23.sql b/swh/scheduler/sql/upgrades/23.sql similarity index 100% rename from sql/updates/23.sql rename to swh/scheduler/sql/upgrades/23.sql diff --git a/sql/updates/24.sql b/swh/scheduler/sql/upgrades/24.sql similarity index 100% rename from sql/updates/24.sql rename to swh/scheduler/sql/upgrades/24.sql diff --git a/sql/updates/25.sql b/swh/scheduler/sql/upgrades/25.sql similarity index 100% rename from sql/updates/25.sql rename to swh/scheduler/sql/upgrades/25.sql diff --git a/sql/updates/26.sql b/swh/scheduler/sql/upgrades/26.sql similarity index 100% rename from sql/updates/26.sql rename to swh/scheduler/sql/upgrades/26.sql diff --git a/sql/updates/27.sql b/swh/scheduler/sql/upgrades/27.sql similarity index 100% rename from sql/updates/27.sql rename to swh/scheduler/sql/upgrades/27.sql diff --git a/sql/updates/28.sql b/swh/scheduler/sql/upgrades/28.sql similarity index 100% rename from sql/updates/28.sql rename to swh/scheduler/sql/upgrades/28.sql diff --git a/sql/updates/29.sql b/swh/scheduler/sql/upgrades/29.sql similarity index 100% rename from sql/updates/29.sql rename to swh/scheduler/sql/upgrades/29.sql diff --git a/sql/updates/30-bis.sql b/swh/scheduler/sql/upgrades/30-bis.sql similarity index 100% rename from sql/updates/30-bis.sql rename to swh/scheduler/sql/upgrades/30-bis.sql diff --git a/sql/updates/30.sql b/swh/scheduler/sql/upgrades/30.sql similarity index 100% rename from sql/updates/30.sql rename to swh/scheduler/sql/upgrades/30.sql diff --git a/sql/updates/31.sql b/swh/scheduler/sql/upgrades/31.sql similarity index 100% rename from sql/updates/31.sql rename to swh/scheduler/sql/upgrades/31.sql diff --git a/sql/updates/32.sql b/swh/scheduler/sql/upgrades/32.sql similarity index 100% rename from sql/updates/32.sql rename to swh/scheduler/sql/upgrades/32.sql diff --git a/swh/scheduler/sql/upgrades/33.sql b/swh/scheduler/sql/upgrades/33.sql new file mode 100644 index 0000000..268b838 --- /dev/null +++ b/swh/scheduler/sql/upgrades/33.sql @@ -0,0 +1,145 @@ +-- SWH DB schema upgrade +-- from_version: 32 +-- to_version: 33 +-- description: Archive old task/task_run tasks + +insert into dbversion (version, release, description) + values (33, now(), 'Work In Progress'); + +-------------------- +-- Schema adaptation +-------------------- + +-- This takes care of copying everything that matters from the old table to the new +-- (sequences, comments...). Keeping what could slow down the copy on the side (indexes, +-- constraints). Those will be installed at the end of the migration. + +create table new_task ( + like task + including all + excluding indexes + excluding constraints +); + +create table new_task_run ( + like task_run + including all + excluding indexes + excluding constraints +); + +------------------------ +-- Actual data migration +------------------------ + +insert into new_task(id, type, arguments, next_run, current_interval, status, policy, + retries_left, priority) +select id, type, arguments, next_run, current_interval, status, policy, + retries_left, priority +from task +where ( + policy='recurring' and (type = 'load-nixguix' or + type like 'list-%' or + type like 'index-%') + ) or ( + policy = 'oneshot' and next_run > now() - interval '2 months' + ); + +-- Keep only the necessary task runs (reusing the previous filtered task entries to +-- create a consistency dataset faster) +insert into new_task_run +select id, task, backend_id, scheduled, started, ended, metadata, status +from task_run where task in ( + select distinct id from new_task +); + +select last_value from task_id_seq; +select last_value from task_run_id_seq; + +----------- +-- Renaming +----------- + +-- Rename current tables to archive_ prefixed names tables +alter table task rename to archive_task; +alter table task_run rename to archive_task_run; + +-- Rename new tables to standard tables +alter table new_task rename to task; +alter table new_task_run rename to task_run; + +----------------------------- +-- Check sequence consistency +----------------------------- + +select last_value from task_id_seq; +select last_value from task_run_id_seq; + +-------------- +-- PKs and FKs +-------------- + +alter table task +add primary key(id); + +alter table task +alter column type set not null; + +alter table task +add constraint task_type_fk +foreign key (type) references task_type (type); + +alter table task +add constraint task_priority_fk +foreign key (priority) references priority_ratio (id); + +alter table task +add constraint task_check_policy +check (policy <> 'recurring' or current_interval is not null) +not valid; + +alter table task + validate constraint task_check_policy; + +alter table task_run +add primary key(id); + +alter table task_run +alter column status set not null, +alter column status set default 'scheduled'; + +alter table task_run +alter column task set not null, +add constraint task_id_fk +foreign key (task) references task (id); + +---------- +-- Indexes +---------- + +create index on task(type); +create index on task(next_run); + +create index on task using btree(type, md5(arguments::text)); +create index on task(priority); + +create index on task_run(task); +create index on task_run(backend_id); + +create index on task_run(task asc, started asc); + +create index on task(type, next_run) +where status = 'next_run_not_scheduled'::task_status; + +---------- +-- Trigger +---------- + +-- Drop trigger on archive table +drop trigger update_task_on_task_end on archive_task_run; + +create trigger update_task_on_task_end + after update of status on task_run + for each row + when (new.status NOT IN ('scheduled', 'started')) + execute procedure swh_scheduler_update_task_on_task_end (); diff --git a/swh/scheduler/tests/common.py b/swh/scheduler/tests/common.py index 6d5da0f..1dc8324 100644 --- a/swh/scheduler/tests/common.py +++ b/swh/scheduler/tests/common.py @@ -1,125 +1,125 @@ # Copyright (C) 2017-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import copy import datetime from typing import Dict, List, Optional TEMPLATES = { - "git": { - "type": "load-git", + "test-git": { + "type": "load-test-git", "arguments": {"args": [], "kwargs": {},}, "next_run": None, }, - "hg": { - "type": "load-hg", + "test-hg": { + "type": "load-test-hg", "arguments": {"args": [], "kwargs": {},}, "next_run": None, "policy": "oneshot", }, } TASK_TYPES = { - "git": { - "type": "load-git", + "test-git": { + "type": "load-test-git", "description": "Update a git repository", "backend_name": "swh.loader.git.tasks.UpdateGitRepository", "default_interval": datetime.timedelta(days=64), "min_interval": datetime.timedelta(hours=12), "max_interval": datetime.timedelta(days=64), "backoff_factor": 2, "max_queue_length": None, "num_retries": 7, "retry_delay": datetime.timedelta(hours=2), }, - "hg": { - "type": "load-hg", + "test-hg": { + "type": "load-test-hg", "description": "Update a mercurial repository", "backend_name": "swh.loader.mercurial.tasks.UpdateHgRepository", "default_interval": datetime.timedelta(days=64), "min_interval": datetime.timedelta(hours=12), "max_interval": datetime.timedelta(days=64), "backoff_factor": 2, "max_queue_length": None, "num_retries": 7, "retry_delay": datetime.timedelta(hours=2), }, } def _task_from_template( template: Dict, next_run: datetime.datetime, priority: Optional[str], *args, **kwargs, ) -> Dict: ret = copy.deepcopy(template) ret["next_run"] = next_run if priority: ret["priority"] = priority if args: ret["arguments"]["args"] = list(args) if kwargs: ret["arguments"]["kwargs"] = kwargs return ret def tasks_from_template( template: Dict, max_timestamp: datetime.datetime, num: Optional[int] = None, priority: Optional[str] = None, num_priorities: Dict[Optional[str], int] = {}, ) -> List[Dict]: """Build ``num`` tasks from template """ assert bool(num) != bool(num_priorities), "mutually exclusive" if not num_priorities: assert num is not None # to please mypy num_priorities = {None: num} tasks: List[Dict] = [] for (priority, num) in num_priorities.items(): for _ in range(num): i = len(tasks) tasks.append( _task_from_template( template, max_timestamp - datetime.timedelta(microseconds=i), priority, "argument-%03d" % i, **{"kwarg%03d" % i: "bogus-kwarg"}, ) ) return tasks def tasks_with_priority_from_template( template: Dict, max_timestamp: datetime.datetime, num: int, priority: str ) -> List[Dict]: """Build tasks with priority from template """ return [ _task_from_template( template, max_timestamp - datetime.timedelta(microseconds=i), priority, "argument-%03d" % i, **{"kwarg%03d" % i: "bogus-kwarg"}, ) for i in range(num) ] LISTERS = ( {"name": "github"}, {"name": "gitlab", "instance_name": "gitlab"}, {"name": "gitlab", "instance_name": "freedesktop"}, {"name": "npm"}, {"name": "pypi"}, ) diff --git a/swh/scheduler/tests/conftest.py b/swh/scheduler/tests/conftest.py index 4872516..3645ecd 100644 --- a/swh/scheduler/tests/conftest.py +++ b/swh/scheduler/tests/conftest.py @@ -1,72 +1,72 @@ # Copyright (C) 2016-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime, timezone import os from typing import Dict, List from unittest.mock import patch import pytest from swh.scheduler.model import ListedOrigin, Lister from swh.scheduler.tests.common import LISTERS # make sure we are not fooled by CELERY_ config environment vars for var in [x for x in os.environ.keys() if x.startswith("CELERY")]: os.environ.pop(var) # test_cli tests depends on a en/C locale, so ensure it os.environ["LC_ALL"] = "C.UTF-8" @pytest.fixture def stored_lister(swh_scheduler) -> Lister: """Store a lister in the scheduler and return its information""" return swh_scheduler.get_or_create_lister(**LISTERS[0]) @pytest.fixture def visit_types() -> List[str]: """Possible visit types in `ListedOrigin`s""" - return ["git", "svn"] + return ["test-git", "test-svn"] @pytest.fixture def listed_origins_by_type( stored_lister: Lister, visit_types: List[str] ) -> Dict[str, List[ListedOrigin]]: """A fixed list of `ListedOrigin`s, for each `visit_type`.""" count_per_type = 1000 assert stored_lister.id return { visit_type: [ ListedOrigin( lister_id=stored_lister.id, url=f"https://{visit_type}.example.com/{i:04d}", visit_type=visit_type, last_update=datetime( 2020, 6, 15, 16, 0, 0, j * count_per_type + i, tzinfo=timezone.utc ), ) for i in range(count_per_type) ] for j, visit_type in enumerate(visit_types) } @pytest.fixture def listed_origins(listed_origins_by_type) -> List[ListedOrigin]: """Return a (fixed) set of listed origins""" return sum(listed_origins_by_type.values(), []) @pytest.fixture def storage(swh_storage): """An instance of in-memory storage that gets injected into the CLI functions.""" with patch("swh.storage.get_storage") as get_storage_mock: get_storage_mock.return_value = swh_storage yield swh_storage diff --git a/swh/scheduler/tests/test_cli_origin.py b/swh/scheduler/tests/test_cli_origin.py index 27d42f2..edf1197 100644 --- a/swh/scheduler/tests/test_cli_origin.py +++ b/swh/scheduler/tests/test_cli_origin.py @@ -1,157 +1,159 @@ # Copyright (C) 2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Tuple import pytest from swh.scheduler.cli.origin import format_origins from swh.scheduler.tests.common import TASK_TYPES from swh.scheduler.tests.test_cli import invoke as basic_invoke def invoke(scheduler, args: Tuple[str, ...] = (), catch_exceptions: bool = False): return basic_invoke( scheduler, args=["origin", *args], catch_exceptions=catch_exceptions ) def test_cli_origin(swh_scheduler): """Check that swh scheduler origin returns its help text""" result = invoke(swh_scheduler) assert "Commands:" in result.stdout def test_format_origins_basic(listed_origins): listed_origins = listed_origins[:100] basic_output = list(format_origins(listed_origins)) # 1 header line + all origins assert len(basic_output) == len(listed_origins) + 1 no_header_output = list(format_origins(listed_origins, with_header=False)) assert basic_output[1:] == no_header_output def test_format_origins_fields_unknown(listed_origins): listed_origins = listed_origins[:10] it = format_origins(listed_origins, fields=["unknown_field"]) with pytest.raises(ValueError, match="unknown_field"): next(it) def test_format_origins_fields(listed_origins): listed_origins = listed_origins[:10] fields = ["lister_id", "url", "visit_type"] output = list(format_origins(listed_origins, fields=fields)) assert output[0] == ",".join(fields) for i, origin in enumerate(listed_origins): assert output[i + 1] == f"{origin.lister_id},{origin.url},{origin.visit_type}" def test_grab_next(swh_scheduler, listed_origins_by_type): NUM_RESULTS = 10 # Strict inequality to check that grab_next_visits doesn't return more # results than requested + + # XXX: should test all of 'listed_origins_by_type' here... visit_type = next(iter(listed_origins_by_type)) assert len(listed_origins_by_type[visit_type]) > NUM_RESULTS for origins in listed_origins_by_type.values(): swh_scheduler.record_listed_origins(origins) result = invoke(swh_scheduler, args=("grab-next", visit_type, str(NUM_RESULTS))) assert result.exit_code == 0 out_lines = result.stdout.splitlines() assert len(out_lines) == NUM_RESULTS + 1 fields = out_lines[0].split(",") returned_origins = [dict(zip(fields, line.split(","))) for line in out_lines[1:]] # Check that we've received origins we had listed in the first place assert set(origin["url"] for origin in returned_origins) <= set( origin.url for origin in listed_origins_by_type[visit_type] ) def test_schedule_next(swh_scheduler, listed_origins_by_type): for task_type in TASK_TYPES.values(): swh_scheduler.create_task_type(task_type) NUM_RESULTS = 10 # Strict inequality to check that grab_next_visits doesn't return more # results than requested visit_type = next(iter(listed_origins_by_type)) assert len(listed_origins_by_type[visit_type]) > NUM_RESULTS for origins in listed_origins_by_type.values(): swh_scheduler.record_listed_origins(origins) result = invoke(swh_scheduler, args=("schedule-next", visit_type, str(NUM_RESULTS))) assert result.exit_code == 0 # pull all tasks out of the scheduler tasks = swh_scheduler.search_tasks() assert len(tasks) == NUM_RESULTS scheduled_tasks = { (task["type"], task["arguments"]["kwargs"]["url"]) for task in tasks } all_possible_tasks = { (f"load-{origin.visit_type}", origin.url) for origin in listed_origins_by_type[visit_type] } assert scheduled_tasks <= all_possible_tasks def test_send_to_celery( mocker, swh_scheduler, swh_scheduler_celery_app, listed_origins_by_type, ): for task_type in TASK_TYPES.values(): swh_scheduler.create_task_type(task_type) visit_type = next(iter(listed_origins_by_type)) for origins in listed_origins_by_type.values(): swh_scheduler.record_listed_origins(origins) get_queue_length = mocker.patch( "swh.scheduler.celery_backend.config.get_queue_length" ) get_queue_length.return_value = None send_task = mocker.patch.object(swh_scheduler_celery_app, "send_task") send_task.return_value = None result = invoke(swh_scheduler, args=("send-to-celery", visit_type)) assert result.exit_code == 0 scheduled_tasks = { (call[0][0], call[1]["kwargs"]["url"]) for call in send_task.call_args_list } expected_tasks = { (TASK_TYPES[origin.visit_type]["backend_name"], origin.url) for origin in listed_origins_by_type[visit_type] } assert expected_tasks == scheduled_tasks def test_update_metrics(swh_scheduler, listed_origins): swh_scheduler.record_listed_origins(listed_origins) assert swh_scheduler.get_metrics() == [] result = invoke(swh_scheduler, args=("update-metrics",)) assert result.exit_code == 0 assert swh_scheduler.get_metrics() != [] diff --git a/swh/scheduler/tests/test_common.py b/swh/scheduler/tests/test_common.py index b5331dc..2439052 100644 --- a/swh/scheduler/tests/test_common.py +++ b/swh/scheduler/tests/test_common.py @@ -1,55 +1,55 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import datetime from .common import TEMPLATES, tasks_from_template def test_tasks_from_template_no_priority(): nb_tasks = 3 - template = TEMPLATES["git"] + template = TEMPLATES["test-git"] next_run = datetime.datetime.utcnow() tasks = tasks_from_template(template, next_run, nb_tasks) assert len(tasks) == nb_tasks for i, t in enumerate(tasks): assert t["type"] == template["type"] assert t["arguments"] is not None assert t.get("policy") is None # not defined in template assert len(t["arguments"]["args"]) == 1 assert len(t["arguments"]["kwargs"].keys()) == 1 assert t["next_run"] == next_run - datetime.timedelta(microseconds=i) assert t.get("priority") is None def test_tasks_from_template_priority(): - template = TEMPLATES["hg"] + template = TEMPLATES["test-hg"] num_priorities = { None: 3, "high": 5, "normal": 3, "low": 2, } next_run = datetime.datetime.utcnow() tasks = tasks_from_template(template, next_run, num_priorities=num_priorities,) assert len(tasks) == sum(num_priorities.values()) repartition_priority = {k: 0 for k in num_priorities} for i, t in enumerate(tasks): assert t["type"] == template["type"] assert t["arguments"] is not None assert t["policy"] == template["policy"] assert len(t["arguments"]["args"]) == 1 assert len(t["arguments"]["kwargs"].keys()) == 1 assert t["next_run"] == next_run - datetime.timedelta(microseconds=i) priority = t.get("priority") assert priority in num_priorities repartition_priority[priority] += 1 assert repartition_priority == num_priorities diff --git a/swh/scheduler/tests/test_recurrent_visits.py b/swh/scheduler/tests/test_recurrent_visits.py index efa8d67..4d83547 100644 --- a/swh/scheduler/tests/test_recurrent_visits.py +++ b/swh/scheduler/tests/test_recurrent_visits.py @@ -1,202 +1,211 @@ # Copyright (C) 2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import timedelta import logging from queue import Queue -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest from swh.scheduler.celery_backend.recurrent_visits import ( POLICY_ADDITIONAL_PARAMETERS, VisitSchedulerThreads, grab_next_visits_policy_weights, send_visits_for_visit_type, spawn_visit_scheduler_thread, terminate_visit_scheduler_threads, visit_scheduler_thread, ) from .test_cli import invoke TEST_MAX_QUEUE = 10000 MODULE_NAME = "swh.scheduler.celery_backend.recurrent_visits" def _compute_backend_name(visit_type: str) -> str: "Build a dummy reproducible backend name" return f"swh.loader.{visit_type}.tasks" @pytest.fixture def swh_scheduler(swh_scheduler): """Override default fixture of the scheduler to install some more task types.""" - for visit_type in ["git", "hg", "svn"]: + for visit_type in ["test-git", "test-hg", "test-svn"]: task_type = f"load-{visit_type}" swh_scheduler.create_task_type( { "type": task_type, "max_queue_length": TEST_MAX_QUEUE, "description": "The {} testing task".format(task_type), "backend_name": _compute_backend_name(visit_type), "default_interval": timedelta(days=1), "min_interval": timedelta(hours=6), "max_interval": timedelta(days=12), } ) return swh_scheduler def test_cli_schedule_recurrent_unknown_visit_type(swh_scheduler): """When passed an unknown visit type, the recurrent visit scheduler should refuse to start.""" with pytest.raises(ValueError, match="Unknown"): invoke( swh_scheduler, False, - ["schedule-recurrent", "--visit-type", "unknown", "--visit-type", "git"], + [ + "schedule-recurrent", + "--visit-type", + "unknown", + "--visit-type", + "test-git", + ], ) def test_cli_schedule_recurrent_noop(swh_scheduler, mocker): """When passing no visit types, the recurrent visit scheduler should start.""" spawn_visit_scheduler_thread = mocker.patch( f"{MODULE_NAME}.spawn_visit_scheduler_thread" ) spawn_visit_scheduler_thread.side_effect = SystemExit # The actual scheduling threads won't spawn, they'll immediately terminate. This # only exercises the logic to pull task types out of the database result = invoke(swh_scheduler, False, ["schedule-recurrent"]) assert result.exit_code == 0, result.output def test_recurrent_visit_scheduling( swh_scheduler, caplog, listed_origins_by_type, mocker, ): """Scheduling known tasks is ok.""" caplog.set_level(logging.DEBUG, MODULE_NAME) nb_origins = 1000 mock_celery_app = MagicMock() mock_available_slots = mocker.patch(f"{MODULE_NAME}.get_available_slots") mock_available_slots.return_value = nb_origins # Slots available in queue # Make sure the scheduler is properly configured in terms of visit/task types all_task_types = { task_type_d["type"]: task_type_d for task_type_d in swh_scheduler.get_task_types() } visit_types = list(listed_origins_by_type.keys()) assert len(visit_types) > 0 task_types = [] origins = [] for visit_type, _origins in listed_origins_by_type.items(): origins.extend(swh_scheduler.record_listed_origins(_origins)) task_type_name = f"load-{visit_type}" assert task_type_name in all_task_types.keys() task_type = all_task_types[task_type_name] task_type["visit_type"] = visit_type # we'll limit the orchestrator to the origins' type we know task_types.append(task_type) - for visit_type in ["git", "svn"]: + for visit_type in ["test-git", "test-svn"]: task_type = f"load-{visit_type}" send_visits_for_visit_type( swh_scheduler, mock_celery_app, visit_type, all_task_types[task_type] ) assert mock_available_slots.called, "The available slots functions should be called" records = [record.message for record in caplog.records] # Mapping over the dict ratio/policies entries can change overall order so let's # check the set of records expected_records = set() for task_type in task_types: visit_type = task_type["visit_type"] queue_name = task_type["backend_name"] msg = ( f"{nb_origins} available slots for visit type {visit_type} " f"in queue {queue_name}" ) expected_records.add(msg) for expected_record in expected_records: assert expected_record in set(records) +@patch.dict( + POLICY_ADDITIONAL_PARAMETERS, {"test-git": POLICY_ADDITIONAL_PARAMETERS["git"]} +) @pytest.mark.parametrize( "visit_type, tablesamples", - [("hg", {}), ("git", POLICY_ADDITIONAL_PARAMETERS["git"])], + [("test-hg", {}), ("test-git", POLICY_ADDITIONAL_PARAMETERS["git"])], ) def test_recurrent_visit_additional_parameters( swh_scheduler, mocker, visit_type, tablesamples ): """Testing additional policy parameters""" mock_grab_next_visits = mocker.patch.object(swh_scheduler, "grab_next_visits") mock_grab_next_visits.return_value = [] grab_next_visits_policy_weights(swh_scheduler, visit_type, 10) for call in mock_grab_next_visits.call_args_list: assert call[1].get("tablesample") == tablesamples.get( call[1]["policy"], {} ).get("tablesample") @pytest.fixture def scheduler_config(swh_scheduler_config): return {"scheduler": {"cls": "local", **swh_scheduler_config}, "celery": {}} def test_visit_scheduler_thread_unknown_task( swh_scheduler, scheduler_config, ): """Starting a thread with unknown task type reports the error""" unknown_visit_type = "unknown" command_queue = Queue() exc_queue = Queue() visit_scheduler_thread( scheduler_config, unknown_visit_type, command_queue, exc_queue ) assert command_queue.empty() is True assert exc_queue.empty() is False assert len(exc_queue.queue) == 1 result = exc_queue.queue.pop() assert result[0] == unknown_visit_type assert isinstance(result[1], ValueError) def test_spawn_visit_scheduler_thread_noop(scheduler_config, visit_types, mocker): """Spawning and terminating threads runs smoothly""" threads: VisitSchedulerThreads = {} exc_queue = Queue() mock_build_app = mocker.patch("swh.scheduler.celery_backend.config.build_app") mock_build_app.return_value = MagicMock() assert len(threads) == 0 for visit_type in visit_types: spawn_visit_scheduler_thread(threads, exc_queue, scheduler_config, visit_type) # This actually only checks the spawning and terminating logic is sound assert len(threads) == len(visit_types) actual_threads = terminate_visit_scheduler_threads(threads) assert not len(actual_threads) assert mock_build_app.called diff --git a/swh/scheduler/tests/test_scheduler.py b/swh/scheduler/tests/test_scheduler.py index 3407084..748ccc6 100644 --- a/swh/scheduler/tests/test_scheduler.py +++ b/swh/scheduler/tests/test_scheduler.py @@ -1,1505 +1,1505 @@ # Copyright (C) 2017-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from collections import defaultdict import copy import datetime from datetime import timedelta import inspect import random from typing import Any, Dict, List, Optional, Tuple import uuid import attr from psycopg2.extras import execute_values import pytest from swh.model.hashutil import hash_to_bytes from swh.scheduler.exc import SchedulerException, StaleData, UnknownPolicy from swh.scheduler.interface import ListedOriginPageToken, SchedulerInterface from swh.scheduler.model import ( LastVisitStatus, ListedOrigin, OriginVisitStats, SchedulerMetrics, ) from swh.scheduler.utils import utcnow from .common import ( LISTERS, TASK_TYPES, TEMPLATES, tasks_from_template, tasks_with_priority_from_template, ) ONEDAY = timedelta(days=1) NUM_PRIORITY_TASKS = {None: 100, "high": 60, "normal": 30, "low": 20} def subdict(d, keys=None, excl=()): if keys is None: keys = [k for k in d.keys()] return {k: d[k] for k in keys if k not in excl} def metrics_sort_key(m: SchedulerMetrics) -> Tuple[uuid.UUID, str]: return (m.lister_id, m.visit_type) def assert_metrics_equal(left, right): assert sorted(left, key=metrics_sort_key) == sorted(right, key=metrics_sort_key) class TestScheduler: def test_interface(self, swh_scheduler): """Checks all methods of SchedulerInterface are implemented by this backend, and that they have the same signature.""" # Create an instance of the protocol (which cannot be instantiated # directly, so this creates a subclass, then instantiates it) interface = type("_", (SchedulerInterface,), {})() assert "create_task_type" in dir(interface) missing_methods = [] for meth_name in dir(interface): if meth_name.startswith("_"): continue interface_meth = getattr(interface, meth_name) try: concrete_meth = getattr(swh_scheduler, meth_name) except AttributeError: if not getattr(interface_meth, "deprecated_endpoint", False): # The backend is missing a (non-deprecated) endpoint missing_methods.append(meth_name) continue expected_signature = inspect.signature(interface_meth) actual_signature = inspect.signature(concrete_meth) assert expected_signature == actual_signature, meth_name assert missing_methods == [] def test_add_task_type(self, swh_scheduler): - tt = TASK_TYPES["git"] + tt = TASK_TYPES["test-git"] swh_scheduler.create_task_type(tt) assert tt == swh_scheduler.get_task_type(tt["type"]) - tt2 = TASK_TYPES["hg"] + tt2 = TASK_TYPES["test-hg"] swh_scheduler.create_task_type(tt2) assert tt == swh_scheduler.get_task_type(tt["type"]) assert tt2 == swh_scheduler.get_task_type(tt2["type"]) def test_create_task_type_idempotence(self, swh_scheduler): - tt = TASK_TYPES["git"] + tt = TASK_TYPES["test-git"] swh_scheduler.create_task_type(tt) swh_scheduler.create_task_type(tt) assert tt == swh_scheduler.get_task_type(tt["type"]) def test_get_task_types(self, swh_scheduler): - tt, tt2 = TASK_TYPES["git"], TASK_TYPES["hg"] + tt, tt2 = TASK_TYPES["test-git"], TASK_TYPES["test-hg"] swh_scheduler.create_task_type(tt) swh_scheduler.create_task_type(tt2) actual_task_types = swh_scheduler.get_task_types() assert tt in actual_task_types assert tt2 in actual_task_types def test_create_tasks(self, swh_scheduler): self._create_task_types(swh_scheduler) num_git = 100 - tasks_1 = tasks_from_template(TEMPLATES["git"], utcnow(), num_git) + tasks_1 = tasks_from_template(TEMPLATES["test-git"], utcnow(), num_git) tasks_2 = tasks_from_template( - TEMPLATES["hg"], utcnow(), num_priorities=NUM_PRIORITY_TASKS + TEMPLATES["test-hg"], utcnow(), num_priorities=NUM_PRIORITY_TASKS ) tasks = tasks_1 + tasks_2 # tasks are returned only once with their ids ret1 = swh_scheduler.create_tasks(tasks + tasks) set_ret1 = set([t["id"] for t in ret1]) # creating the same set result in the same ids ret = swh_scheduler.create_tasks(tasks) set_ret = set([t["id"] for t in ret]) # Idempotence results assert set_ret == set_ret1 assert len(ret) == len(ret1) ids = set() actual_priorities = defaultdict(int) for task, orig_task in zip(ret, tasks): task = copy.deepcopy(task) - task_type = TASK_TYPES[orig_task["type"].split("-")[-1]] + task_type = TASK_TYPES[orig_task["type"].split("-", 1)[-1]] assert task["id"] not in ids assert task["status"] == "next_run_not_scheduled" assert task["current_interval"] == task_type["default_interval"] assert task["policy"] == orig_task.get("policy", "recurring") priority = task.get("priority") actual_priorities[priority] += 1 assert task["retries_left"] == (task_type["num_retries"] or 0) ids.add(task["id"]) del task["id"] del task["status"] del task["current_interval"] del task["retries_left"] if "policy" not in orig_task: del task["policy"] if "priority" not in orig_task: del task["priority"] assert task == orig_task expected_priorities = NUM_PRIORITY_TASKS.copy() expected_priorities[None] += num_git assert dict(actual_priorities) == expected_priorities def test_peek_ready_tasks_no_priority(self, swh_scheduler): self._create_task_types(swh_scheduler) t = utcnow() - task_type = TEMPLATES["git"]["type"] - tasks = tasks_from_template(TEMPLATES["git"], t, 100) + task_type = TEMPLATES["test-git"]["type"] + tasks = tasks_from_template(TEMPLATES["test-git"], t, 100) random.shuffle(tasks) swh_scheduler.create_tasks(tasks) ready_tasks = swh_scheduler.peek_ready_tasks(task_type) assert len(ready_tasks) == len(tasks) for i in range(len(ready_tasks) - 1): assert ready_tasks[i]["next_run"] <= ready_tasks[i + 1]["next_run"] # Only get the first few ready tasks limit = random.randrange(5, 5 + len(tasks) // 2) ready_tasks_limited = swh_scheduler.peek_ready_tasks(task_type, num_tasks=limit) assert len(ready_tasks_limited) == limit assert ready_tasks_limited == ready_tasks[:limit] # Limit by timestamp max_ts = tasks[limit - 1]["next_run"] ready_tasks_timestamped = swh_scheduler.peek_ready_tasks( task_type, timestamp=max_ts ) for ready_task in ready_tasks_timestamped: assert ready_task["next_run"] <= max_ts # Make sure we get proper behavior for the first ready tasks assert ready_tasks[: len(ready_tasks_timestamped)] == ready_tasks_timestamped # Limit by both ready_tasks_both = swh_scheduler.peek_ready_tasks( task_type, timestamp=max_ts, num_tasks=limit // 3 ) assert len(ready_tasks_both) <= limit // 3 for ready_task in ready_tasks_both: assert ready_task["next_run"] <= max_ts assert ready_task in ready_tasks[: limit // 3] def test_peek_ready_tasks_returns_only_no_priority_tasks(self, swh_scheduler): """Peek ready tasks only return standard tasks (no priority)""" self._create_task_types(swh_scheduler) t = utcnow() - task_type = TEMPLATES["git"]["type"] + task_type = TEMPLATES["test-git"]["type"] # Create tasks with and without priorities tasks = tasks_from_template( - TEMPLATES["git"], t, num_priorities=NUM_PRIORITY_TASKS, + TEMPLATES["test-git"], t, num_priorities=NUM_PRIORITY_TASKS, ) count_priority = 0 for task in tasks: count_priority += 0 if task.get("priority") is None else 1 assert count_priority > 0, "Some created tasks should have some priority" random.shuffle(tasks) swh_scheduler.create_tasks(tasks) # take all available no priority tasks ready_tasks = swh_scheduler.peek_ready_tasks(task_type) assert len(ready_tasks) == len(tasks) - count_priority # No read task should have any priority for task in ready_tasks: assert task.get("priority") is None def test_grab_ready_tasks(self, swh_scheduler): self._create_task_types(swh_scheduler) t = utcnow() - task_type = TEMPLATES["git"]["type"] + task_type = TEMPLATES["test-git"]["type"] # Create tasks with and without priorities tasks = tasks_from_template( - TEMPLATES["git"], t, num_priorities=NUM_PRIORITY_TASKS + TEMPLATES["test-git"], t, num_priorities=NUM_PRIORITY_TASKS ) random.shuffle(tasks) swh_scheduler.create_tasks(tasks) first_ready_tasks = swh_scheduler.peek_ready_tasks(task_type, num_tasks=50) grabbed_tasks = swh_scheduler.grab_ready_tasks(task_type, num_tasks=50) first_ready_tasks.sort(key=lambda task: task["arguments"]["args"][0]) grabbed_tasks.sort(key=lambda task: task["arguments"]["args"][0]) for peeked, grabbed in zip(first_ready_tasks, grabbed_tasks): assert peeked["status"] == "next_run_not_scheduled" del peeked["status"] assert grabbed["status"] == "next_run_scheduled" del grabbed["status"] assert peeked == grabbed priority = grabbed["priority"] assert priority == peeked["priority"] assert priority is None def test_grab_ready_priority_tasks(self, swh_scheduler): """check the grab and peek priority tasks endpoint behave as expected""" self._create_task_types(swh_scheduler) t = utcnow() - task_type = TEMPLATES["git"]["type"] + task_type = TEMPLATES["test-git"]["type"] num_tasks = 100 # Create tasks with and without priorities tasks0 = tasks_with_priority_from_template( - TEMPLATES["git"], t, num_tasks, "high", + TEMPLATES["test-git"], t, num_tasks, "high", ) tasks1 = tasks_with_priority_from_template( - TEMPLATES["hg"], t, num_tasks, "low", + TEMPLATES["test-hg"], t, num_tasks, "low", ) tasks2 = tasks_with_priority_from_template( - TEMPLATES["hg"], t, num_tasks, "normal", + TEMPLATES["test-hg"], t, num_tasks, "normal", ) tasks = tasks0 + tasks1 + tasks2 random.shuffle(tasks) swh_scheduler.create_tasks(tasks) ready_tasks = swh_scheduler.peek_ready_priority_tasks(task_type, num_tasks=50) grabbed_tasks = swh_scheduler.grab_ready_priority_tasks(task_type, num_tasks=50) ready_tasks.sort(key=lambda task: task["arguments"]["args"][0]) grabbed_tasks.sort(key=lambda task: task["arguments"]["args"][0]) for peeked, grabbed in zip(ready_tasks, grabbed_tasks): assert peeked["status"] == "next_run_not_scheduled" del peeked["status"] assert grabbed["status"] == "next_run_scheduled" del grabbed["status"] assert peeked == grabbed assert peeked["priority"] == grabbed["priority"] assert peeked["priority"] is not None def test_get_tasks(self, swh_scheduler): self._create_task_types(swh_scheduler) t = utcnow() - tasks = tasks_from_template(TEMPLATES["git"], t, 100) + tasks = tasks_from_template(TEMPLATES["test-git"], t, 100) tasks = swh_scheduler.create_tasks(tasks) random.shuffle(tasks) while len(tasks) > 1: length = random.randrange(1, len(tasks)) cur_tasks = sorted(tasks[:length], key=lambda x: x["id"]) tasks[:length] = [] ret = swh_scheduler.get_tasks(task["id"] for task in cur_tasks) # result is not guaranteed to be sorted ret.sort(key=lambda x: x["id"]) assert ret == cur_tasks def test_search_tasks(self, swh_scheduler): def make_real_dicts(lst): """RealDictRow is not a real dict.""" return [dict(d.items()) for d in lst] self._create_task_types(swh_scheduler) t = utcnow() - tasks = tasks_from_template(TEMPLATES["git"], t, 100) + tasks = tasks_from_template(TEMPLATES["test-git"], t, 100) tasks = swh_scheduler.create_tasks(tasks) assert make_real_dicts(swh_scheduler.search_tasks()) == make_real_dicts(tasks) def assert_filtered_task_ok( self, task: Dict[str, Any], after: datetime.datetime, before: datetime.datetime ) -> None: """Ensure filtered tasks have the right expected properties (within the range, recurring disabled, etc..) """ started = task["started"] date = started if started is not None else task["scheduled"] assert after <= date and date <= before if task["task_policy"] == "oneshot": assert task["task_status"] in ["completed", "disabled"] if task["task_policy"] == "recurring": assert task["task_status"] in ["disabled"] def test_filter_task_to_archive(self, swh_scheduler): """Filtering only list disabled recurring or completed oneshot tasks """ self._create_task_types(swh_scheduler) _time = utcnow() - recurring = tasks_from_template(TEMPLATES["git"], _time, 12) - oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12) + recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12) + oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12) total_tasks = len(recurring) + len(oneshots) # simulate scheduling tasks pending_tasks = swh_scheduler.create_tasks(recurring + oneshots) backend_tasks = [ { "task": task["id"], "backend_id": str(uuid.uuid4()), "scheduled": utcnow(), } for task in pending_tasks ] swh_scheduler.mass_schedule_task_runs(backend_tasks) # we simulate the task are being done _tasks = [] for task in backend_tasks: t = swh_scheduler.end_task_run(task["backend_id"], status="eventful") _tasks.append(t) # Randomly update task's status per policy status_per_policy = {"recurring": 0, "oneshot": 0} status_choice = { # policy: [tuple (1-for-filtering, 'associated-status')] "recurring": [ (1, "disabled"), (0, "completed"), (0, "next_run_not_scheduled"), ], "oneshot": [ (0, "next_run_not_scheduled"), (1, "disabled"), (1, "completed"), ], } tasks_to_update = defaultdict(list) _task_ids = defaultdict(list) # randomize 'disabling' recurring task or 'complete' oneshot task for task in pending_tasks: policy = task["policy"] _task_ids[policy].append(task["id"]) status = random.choice(status_choice[policy]) if status[0] != 1: continue # elected for filtering status_per_policy[policy] += status[0] tasks_to_update[policy].append(task["id"]) swh_scheduler.disable_tasks(tasks_to_update["recurring"]) # hack: change the status to something else than completed/disabled swh_scheduler.set_status_tasks( _task_ids["oneshot"], status="next_run_not_scheduled" ) # complete the tasks to update swh_scheduler.set_status_tasks(tasks_to_update["oneshot"], status="completed") total_tasks_filtered = ( status_per_policy["recurring"] + status_per_policy["oneshot"] ) # no pagination scenario # retrieve tasks to archive after = _time - ONEDAY after_ts = after.strftime("%Y-%m-%d") before = utcnow() + ONEDAY before_ts = before.strftime("%Y-%m-%d") tasks_result = swh_scheduler.filter_task_to_archive( after_ts=after_ts, before_ts=before_ts, limit=total_tasks ) tasks_to_archive = tasks_result["tasks"] assert len(tasks_to_archive) == total_tasks_filtered assert tasks_result.get("next_page_token") is None actual_filtered_per_status = {"recurring": 0, "oneshot": 0} for task in tasks_to_archive: self.assert_filtered_task_ok(task, after, before) actual_filtered_per_status[task["task_policy"]] += 1 assert actual_filtered_per_status == status_per_policy # pagination scenario nb_tasks = 3 tasks_result = swh_scheduler.filter_task_to_archive( after_ts=after_ts, before_ts=before_ts, limit=nb_tasks ) tasks_to_archive2 = tasks_result["tasks"] assert len(tasks_to_archive2) == nb_tasks next_page_token = tasks_result["next_page_token"] assert next_page_token is not None all_tasks = tasks_to_archive2 while next_page_token is not None: # Retrieve paginated results tasks_result = swh_scheduler.filter_task_to_archive( after_ts=after_ts, before_ts=before_ts, limit=nb_tasks, page_token=next_page_token, ) tasks_to_archive2 = tasks_result["tasks"] assert len(tasks_to_archive2) <= nb_tasks all_tasks.extend(tasks_to_archive2) next_page_token = tasks_result.get("next_page_token") actual_filtered_per_status = {"recurring": 0, "oneshot": 0} for task in all_tasks: self.assert_filtered_task_ok(task, after, before) actual_filtered_per_status[task["task_policy"]] += 1 assert actual_filtered_per_status == status_per_policy def test_delete_archived_tasks(self, swh_scheduler): self._create_task_types(swh_scheduler) _time = utcnow() - recurring = tasks_from_template(TEMPLATES["git"], _time, 12) - oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12) + recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12) + oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12) total_tasks = len(recurring) + len(oneshots) pending_tasks = swh_scheduler.create_tasks(recurring + oneshots) backend_tasks = [ { "task": task["id"], "backend_id": str(uuid.uuid4()), "scheduled": utcnow(), } for task in pending_tasks ] swh_scheduler.mass_schedule_task_runs(backend_tasks) _tasks = [] percent = random.randint(0, 100) # random election removal boundary for task in backend_tasks: t = swh_scheduler.end_task_run(task["backend_id"], status="eventful") c = random.randint(0, 100) if c <= percent: _tasks.append({"task_id": t["task"], "task_run_id": t["id"]}) swh_scheduler.delete_archived_tasks(_tasks) all_tasks = [task["id"] for task in swh_scheduler.search_tasks()] tasks_count = len(all_tasks) tasks_run_count = len(swh_scheduler.get_task_runs(all_tasks)) assert tasks_count == total_tasks - len(_tasks) assert tasks_run_count == total_tasks - len(_tasks) def test_get_task_runs_no_task(self, swh_scheduler): """No task exist in the scheduler's db, get_task_runs() should always return an empty list. """ assert not swh_scheduler.get_task_runs(task_ids=()) assert not swh_scheduler.get_task_runs(task_ids=(1, 2, 3)) assert not swh_scheduler.get_task_runs(task_ids=(1, 2, 3), limit=10) def test_get_task_runs_no_task_executed(self, swh_scheduler): """No task has been executed yet, get_task_runs() should always return an empty list. """ self._create_task_types(swh_scheduler) _time = utcnow() - recurring = tasks_from_template(TEMPLATES["git"], _time, 12) - oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12) + recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12) + oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12) swh_scheduler.create_tasks(recurring + oneshots) assert not swh_scheduler.get_task_runs(task_ids=()) assert not swh_scheduler.get_task_runs(task_ids=(1, 2, 3)) assert not swh_scheduler.get_task_runs(task_ids=(1, 2, 3), limit=10) def test_get_task_runs_with_scheduled(self, swh_scheduler): """Some tasks have been scheduled but not executed yet, get_task_runs() should not return an empty list. limit should behave as expected. """ self._create_task_types(swh_scheduler) _time = utcnow() - recurring = tasks_from_template(TEMPLATES["git"], _time, 12) - oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12) + recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12) + oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12) total_tasks = len(recurring) + len(oneshots) pending_tasks = swh_scheduler.create_tasks(recurring + oneshots) backend_tasks = [ { "task": task["id"], "backend_id": str(uuid.uuid4()), "scheduled": utcnow(), } for task in pending_tasks ] swh_scheduler.mass_schedule_task_runs(backend_tasks) assert not swh_scheduler.get_task_runs(task_ids=[total_tasks + 1]) btask = backend_tasks[0] runs = swh_scheduler.get_task_runs(task_ids=[btask["task"]]) assert len(runs) == 1 run = runs[0] assert subdict(run, excl=("id",)) == { "task": btask["task"], "backend_id": btask["backend_id"], "scheduled": btask["scheduled"], "started": None, "ended": None, "metadata": None, "status": "scheduled", } runs = swh_scheduler.get_task_runs( task_ids=[bt["task"] for bt in backend_tasks], limit=2 ) assert len(runs) == 2 runs = swh_scheduler.get_task_runs( task_ids=[bt["task"] for bt in backend_tasks] ) assert len(runs) == total_tasks keys = ("task", "backend_id", "scheduled") assert ( sorted([subdict(x, keys) for x in runs], key=lambda x: x["task"]) == backend_tasks ) def test_get_task_runs_with_executed(self, swh_scheduler): """Some tasks have been executed, get_task_runs() should not return an empty list. limit should behave as expected. """ self._create_task_types(swh_scheduler) _time = utcnow() - recurring = tasks_from_template(TEMPLATES["git"], _time, 12) - oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12) + recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12) + oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12) pending_tasks = swh_scheduler.create_tasks(recurring + oneshots) backend_tasks = [ { "task": task["id"], "backend_id": str(uuid.uuid4()), "scheduled": utcnow(), } for task in pending_tasks ] swh_scheduler.mass_schedule_task_runs(backend_tasks) btask = backend_tasks[0] ts = utcnow() swh_scheduler.start_task_run( btask["backend_id"], metadata={"something": "stupid"}, timestamp=ts ) runs = swh_scheduler.get_task_runs(task_ids=[btask["task"]]) assert len(runs) == 1 assert subdict(runs[0], excl=("id")) == { "task": btask["task"], "backend_id": btask["backend_id"], "scheduled": btask["scheduled"], "started": ts, "ended": None, "metadata": {"something": "stupid"}, "status": "started", } ts2 = utcnow() swh_scheduler.end_task_run( btask["backend_id"], metadata={"other": "stuff"}, timestamp=ts2, status="eventful", ) runs = swh_scheduler.get_task_runs(task_ids=[btask["task"]]) assert len(runs) == 1 assert subdict(runs[0], excl=("id")) == { "task": btask["task"], "backend_id": btask["backend_id"], "scheduled": btask["scheduled"], "started": ts, "ended": ts2, "metadata": {"something": "stupid", "other": "stuff"}, "status": "eventful", } def test_get_or_create_lister(self, swh_scheduler): db_listers = [] for lister_args in LISTERS: db_listers.append(swh_scheduler.get_or_create_lister(**lister_args)) for lister, lister_args in zip(db_listers, LISTERS): assert lister.name == lister_args["name"] assert lister.instance_name == lister_args.get("instance_name", "") lister_get_again = swh_scheduler.get_or_create_lister( lister.name, lister.instance_name ) assert lister == lister_get_again def test_get_lister(self, swh_scheduler): for lister_args in LISTERS: assert swh_scheduler.get_lister(**lister_args) is None db_listers = [] for lister_args in LISTERS: db_listers.append(swh_scheduler.get_or_create_lister(**lister_args)) for lister, lister_args in zip(db_listers, LISTERS): lister_get_again = swh_scheduler.get_lister( lister.name, lister.instance_name ) assert lister == lister_get_again def test_get_listers(self, swh_scheduler): assert swh_scheduler.get_listers() == [] db_listers = [] for lister_args in LISTERS: db_listers.append(swh_scheduler.get_or_create_lister(**lister_args)) assert swh_scheduler.get_listers() == db_listers def test_update_lister(self, swh_scheduler, stored_lister): lister = attr.evolve(stored_lister, current_state={"updated": "now"}) updated_lister = swh_scheduler.update_lister(lister) assert updated_lister.updated > lister.updated assert updated_lister == attr.evolve(lister, updated=updated_lister.updated) def test_update_lister_stale(self, swh_scheduler, stored_lister): swh_scheduler.update_lister(stored_lister) with pytest.raises(StaleData) as exc: swh_scheduler.update_lister(stored_lister) assert "state not updated" in exc.value.args[0] def test_record_listed_origins(self, swh_scheduler, listed_origins): ret = swh_scheduler.record_listed_origins(listed_origins) assert set(returned.url for returned in ret) == set( origin.url for origin in listed_origins ) assert all(origin.first_seen == origin.last_seen for origin in ret) def test_record_listed_origins_with_duplicate(self, swh_scheduler, listed_origins): # the duplicates must be in the same page to raise the "on conflict error" listed_origins.insert(0, listed_origins[0]) ret = swh_scheduler.record_listed_origins(listed_origins) # without the duplicate assert len(ret) == len(listed_origins) - 1 def test_record_listed_origins_upsert(self, swh_scheduler, listed_origins): # First, insert `cutoff` origins cutoff = 100 assert cutoff < len(listed_origins) ret = swh_scheduler.record_listed_origins(listed_origins[:cutoff]) assert len(ret) == cutoff # Then, insert all origins, including the `cutoff` first. ret = swh_scheduler.record_listed_origins(listed_origins) assert len(ret) == len(listed_origins) # Two different "first seen" values assert len(set(origin.first_seen for origin in ret)) == 2 # But a single "last seen" value assert len(set(origin.last_seen for origin in ret)) == 1 def test_get_listed_origins_exact(self, swh_scheduler, listed_origins): swh_scheduler.record_listed_origins(listed_origins) for i, origin in enumerate(listed_origins): ret = swh_scheduler.get_listed_origins( lister_id=origin.lister_id, url=origin.url ) assert ret.next_page_token is None assert len(ret.results) == 1 assert ret.results[0].lister_id == origin.lister_id assert ret.results[0].url == origin.url @pytest.mark.parametrize("num_origins,limit", [(20, 6), (5, 42), (20, 20)]) def test_get_listed_origins_limit( self, swh_scheduler, listed_origins, num_origins, limit ) -> None: added_origins = sorted( listed_origins[:num_origins], key=lambda o: (o.lister_id, o.url) ) swh_scheduler.record_listed_origins(added_origins) returned_origins: List[ListedOrigin] = [] call_count = 0 next_page_token: Optional[ListedOriginPageToken] = None while True: call_count += 1 ret = swh_scheduler.get_listed_origins( lister_id=listed_origins[0].lister_id, limit=limit, page_token=next_page_token, ) returned_origins.extend(ret.results) next_page_token = ret.next_page_token if next_page_token is None: break assert call_count == (num_origins // limit) + 1 assert len(returned_origins) == num_origins assert [(origin.lister_id, origin.url) for origin in returned_origins] == [ (origin.lister_id, origin.url) for origin in added_origins ] def test_get_listed_origins_all(self, swh_scheduler, listed_origins) -> None: swh_scheduler.record_listed_origins(listed_origins) ret = swh_scheduler.get_listed_origins(limit=len(listed_origins) + 1) assert ret.next_page_token is None assert len(ret.results) == len(listed_origins) def _grab_next_visits_setup(self, swh_scheduler, listed_origins_by_type): """Basic origins setup for scheduling policy tests""" visit_type = next(iter(listed_origins_by_type)) origins = listed_origins_by_type[visit_type][:100] assert len(origins) > 0 recorded_origins = swh_scheduler.record_listed_origins(origins) return visit_type, recorded_origins def _check_grab_next_visit_basic( self, swh_scheduler, visit_type, policy, expected, **kwargs ): """Calls grab_next_visits with the passed policy, and check that: - all the origins returned are the expected ones (in the same order) - no extra origins are returned - the last_scheduled field has been set properly. Pass the extra keyword arguments to the calls to grab_next_visits. Returns a timestamp greater than all `last_scheduled` values for the grabbed visits. """ assert len(expected) != 0 before = utcnow() ret = swh_scheduler.grab_next_visits( visit_type=visit_type, # Request one more than expected to check that no extra origin is returned count=len(expected) + 1, policy=policy, **kwargs, ) after = utcnow() assert ret == expected visit_stats_list = swh_scheduler.origin_visit_stats_get( [(origin.url, origin.visit_type) for origin in expected] ) assert len(visit_stats_list) == len(expected) for visit_stats in visit_stats_list: # Check that last_scheduled got updated assert before <= visit_stats.last_scheduled <= after # They should not be scheduled again ret = swh_scheduler.grab_next_visits( visit_type=visit_type, count=len(expected) + 1, policy=policy, **kwargs ) assert ret == [], "grab_next_visits returned already-scheduled origins" return after def _check_grab_next_visit( self, swh_scheduler, visit_type, policy, expected, **kwargs ): """Run the same check as _check_grab_next_visit_basic, but also checks the origin visits have been marked as scheduled, and are only re-scheduled a week later """ after = self._check_grab_next_visit_basic( swh_scheduler, visit_type, policy, expected, **kwargs ) # But a week, later, they should ret = swh_scheduler.grab_next_visits( visit_type=visit_type, count=len(expected) + 1, policy=policy, timestamp=after + timedelta(days=7), ) # We need to sort them because their 'last_scheduled' field is updated to # exactly the same value, so the order is not deterministic assert sorted(ret) == sorted( expected ), "grab_next_visits didn't reschedule visits after a week" def _prepare_oldest_scheduled_first_origins( self, swh_scheduler, listed_origins_by_type ): visit_type, origins = self._grab_next_visits_setup( swh_scheduler, listed_origins_by_type ) # Give all origins but one a last_scheduled date base_date = datetime.datetime(2020, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc) visit_stats = [ OriginVisitStats( url=origin.url, visit_type=origin.visit_type, last_snapshot=None, last_successful=None, last_visit=None, last_scheduled=base_date - timedelta(seconds=i), ) for i, origin in enumerate(origins[1:]) ] swh_scheduler.origin_visit_stats_upsert(visit_stats) # We expect to retrieve the origin with a NULL last_scheduled # as well as those with the oldest values (i.e. the last ones), in order. expected = [origins[0]] + origins[1:][::-1] return visit_type, origins, expected def test_grab_next_visits_oldest_scheduled_first( self, swh_scheduler, listed_origins_by_type, ): visit_type, origins, expected = self._prepare_oldest_scheduled_first_origins( swh_scheduler, listed_origins_by_type ) self._check_grab_next_visit( swh_scheduler, visit_type=visit_type, policy="oldest_scheduled_first", expected=expected, ) @pytest.mark.parametrize("which_cooldown", ("scheduled", "failed", "not_found")) @pytest.mark.parametrize("cooldown", (7, 15)) def test_grab_next_visits_cooldowns( self, swh_scheduler, listed_origins_by_type, which_cooldown, cooldown, ): visit_type, origins, expected = self._prepare_oldest_scheduled_first_origins( swh_scheduler, listed_origins_by_type ) after = self._check_grab_next_visit_basic( swh_scheduler, visit_type=visit_type, policy="oldest_scheduled_first", expected=expected, ) # Mark all the visits as scheduled, failed or notfound on the `after` timestamp ovs_args = { "last_visit": None, "last_visit_status": None, "last_scheduled": None, } if which_cooldown == "scheduled": ovs_args["last_scheduled"] = after else: ovs_args["last_visit"] = after ovs_args["last_visit_status"] = LastVisitStatus(which_cooldown) visit_stats = [ OriginVisitStats( url=origin.url, visit_type=origin.visit_type, last_snapshot=None, last_successful=None, **ovs_args, ) for i, origin in enumerate(origins) ] swh_scheduler.origin_visit_stats_upsert(visit_stats) cooldown_td = timedelta(days=cooldown) cooldown_args = { "scheduled_cooldown": None, "failed_cooldown": None, "not_found_cooldown": None, } cooldown_args[f"{which_cooldown}_cooldown"] = cooldown_td ret = swh_scheduler.grab_next_visits( visit_type=visit_type, count=len(expected) + 1, policy="oldest_scheduled_first", timestamp=after + cooldown_td - timedelta(seconds=1), **cooldown_args, ) assert ret == [], f"{which_cooldown}_cooldown ignored" ret = swh_scheduler.grab_next_visits( visit_type=visit_type, count=len(expected) + 1, policy="oldest_scheduled_first", timestamp=after + cooldown_td + timedelta(seconds=1), **cooldown_args, ) assert sorted(ret) == sorted( expected ), "grab_next_visits didn't reschedule visits after the configured cooldown" def test_grab_next_visits_tablesample( self, swh_scheduler, listed_origins_by_type, ): visit_type, origins, expected = self._prepare_oldest_scheduled_first_origins( swh_scheduler, listed_origins_by_type ) ret = swh_scheduler.grab_next_visits( visit_type=visit_type, policy="oldest_scheduled_first", tablesample=50, count=len(expected), ) # Just a smoke test, not obvious how to test this more reliably assert ret is not None def test_grab_next_visits_never_visited_oldest_update_first( self, swh_scheduler, listed_origins_by_type, ): visit_type, origins = self._grab_next_visits_setup( swh_scheduler, listed_origins_by_type ) # Update known origins with a `last_update` field that we control base_date = datetime.datetime(2020, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc) updated_origins = [ attr.evolve(origin, last_update=base_date - timedelta(seconds=i)) for i, origin in enumerate(origins) ] updated_origins = swh_scheduler.record_listed_origins(updated_origins) # We expect to retrieve origins with the oldest update date, that is # origins at the end of our updated_origins list. expected_origins = sorted(updated_origins, key=lambda o: o.last_update) self._check_grab_next_visit( swh_scheduler, visit_type=visit_type, policy="never_visited_oldest_update_first", expected=expected_origins, ) def test_grab_next_visits_already_visited_order_by_lag( self, swh_scheduler, listed_origins_by_type, ): visit_type, origins = self._grab_next_visits_setup( swh_scheduler, listed_origins_by_type ) # Update known origins with a `last_update` field that we control base_date = datetime.datetime(2020, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc) updated_origins = [ attr.evolve(origin, last_update=base_date - timedelta(seconds=i)) for i, origin in enumerate(origins) ] updated_origins = swh_scheduler.record_listed_origins(updated_origins) # Update the visit stats with a known visit at a controlled date for # half the origins. Pick the date in the middle of the # updated_origins' `last_update` range visit_date = updated_origins[len(updated_origins) // 2].last_update visited_origins = updated_origins[::2] visit_stats = [ OriginVisitStats( url=origin.url, visit_type=origin.visit_type, last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"), last_successful=visit_date, last_visit=visit_date, ) for origin in visited_origins ] swh_scheduler.origin_visit_stats_upsert(visit_stats) # We expect to retrieve visited origins with the largest lag, but only # those which haven't been visited since their last update expected_origins = sorted( [origin for origin in visited_origins if origin.last_update > visit_date], key=lambda o: visit_date - o.last_update, ) self._check_grab_next_visit( swh_scheduler, visit_type=visit_type, policy="already_visited_order_by_lag", expected=expected_origins, ) def test_grab_next_visits_underflow(self, swh_scheduler, listed_origins_by_type): """Check that grab_next_visits works when there not enough origins in the database""" visit_type = next(iter(listed_origins_by_type)) # Only add 5 origins to the database origins = listed_origins_by_type[visit_type][:5] assert origins swh_scheduler.record_listed_origins(origins) ret = swh_scheduler.grab_next_visits( visit_type, len(origins) + 2, policy="oldest_scheduled_first" ) assert len(ret) == 5 def test_grab_next_visits_no_last_update_nor_visit_stats( self, swh_scheduler, listed_origins_by_type ): """grab_next_visits should retrieve tasks without last update (nor visit stats) """ visit_type = next(iter(listed_origins_by_type)) origins = [] for origin in listed_origins_by_type[visit_type]: origins.append( attr.evolve(origin, last_update=None) ) # void the last update so we are in the relevant context assert len(origins) > 0 swh_scheduler.record_listed_origins(origins) # Initially, we have no global queue position current_state = swh_scheduler.visit_scheduler_queue_position_get() assert current_state == {} # nor any visit statuses actual_visit_stats = swh_scheduler.origin_visit_stats_get( (o.url, o.visit_type) for o in origins ) assert len(actual_visit_stats) == 0 # Grab some new visits next_visits = swh_scheduler.grab_next_visits( visit_type, count=len(origins), policy="origins_without_last_update", ) # we do have the one without any last update assert len(next_visits) == len(origins) # Now the global state got updated current_state = swh_scheduler.visit_scheduler_queue_position_get() assert current_state[visit_type] is not None actual_visit_stats = swh_scheduler.origin_visit_stats_get( (o.url, o.visit_type) for o in next_visits ) # Visit stats got algo created assert len(actual_visit_stats) == len(origins) def test_grab_next_visits_no_last_update_with_visit_stats( self, swh_scheduler, listed_origins_by_type ): """grab_next_visits should retrieve tasks without last update""" visit_type = next(iter(listed_origins_by_type)) origins = [] for origin in listed_origins_by_type[visit_type]: origins.append( attr.evolve(origin, last_update=None) ) # void the last update so we are in the relevant context assert len(origins) > 0 swh_scheduler.record_listed_origins(origins) # Initially, we have no global queue position current_state = swh_scheduler.visit_scheduler_queue_position_get() assert current_state == {} # Simulate some of those origins have associated visit stats (some with an # existing queue position and some without any) visit_stats = ( [ OriginVisitStats( url=origin.url, visit_type=origin.visit_type, last_successful=utcnow(), last_visit=utcnow(), next_visit_queue_position=int(24 * 3600 * random.uniform(-10, 1)), ) for origin in origins[:100] ] + [ OriginVisitStats( url=origin.url, visit_type=origin.visit_type, last_successful=utcnow(), last_visit=utcnow(), next_visit_queue_position=int( 24 * 3600 * random.uniform(1, 10) ), # definitely > 0 ) for origin in origins[100:150] ] + [ OriginVisitStats( url=origin.url, visit_type=origin.visit_type, last_successful=utcnow(), last_visit=utcnow(), ) for origin in origins[150:] ] ) swh_scheduler.origin_visit_stats_upsert(visit_stats) # Grab next visits actual_visits = swh_scheduler.grab_next_visits( visit_type, count=len(origins), policy="origins_without_last_update", ) assert len(actual_visits) == len(origins) actual_visit_stats = swh_scheduler.origin_visit_stats_get( (o.url, o.visit_type) for o in actual_visits ) assert len(actual_visit_stats) == len(origins) current_state = swh_scheduler.visit_scheduler_queue_position_get() assert current_state == { visit_type: max( s.next_visit_queue_position for s in actual_visit_stats if s.next_visit_queue_position is not None ) } def test_grab_next_visits_unknown_policy(self, swh_scheduler): unknown_policy = "non_existing_policy" NUM_RESULTS = 5 with pytest.raises(UnknownPolicy, match=unknown_policy): swh_scheduler.grab_next_visits("type", NUM_RESULTS, policy=unknown_policy) def test_grab_next_visit_duplicates(self, swh_scheduler, listed_origins): """Checks grab_next_visits does not crash when there are rows with duplicated (origin_url, visit_type) in the database """ lister2 = swh_scheduler.get_or_create_lister(**LISTERS[1]) assert lister2.id != listed_origins[0].lister_id # Create two origins with the same url and visit_type, but different listers # (and also differing value for last_update so they are returned in # deterministic order) origin1 = attr.evolve( listed_origins[0], first_seen=utcnow(), last_seen=utcnow() ) origin2 = attr.evolve( origin1, lister_id=lister2.id, last_update=origin1.last_update + datetime.timedelta(seconds=10), ) origins = [origin1, origin2] recorded_origins = swh_scheduler.record_listed_origins(origins) expected_origins = sorted(recorded_origins, key=lambda o: o.last_update) self._check_grab_next_visit( swh_scheduler, visit_type=origin1.visit_type, policy="never_visited_oldest_update_first", expected=expected_origins, ) def _create_task_types(self, scheduler): for tt in TASK_TYPES.values(): scheduler.create_task_type(tt) def test_origin_visit_stats_get_empty(self, swh_scheduler) -> None: assert swh_scheduler.origin_visit_stats_get([]) == [] def test_origin_visit_stats_get_pagination(self, swh_scheduler) -> None: page_size = inspect.signature(execute_values).parameters["page_size"].default visit_stats = [ OriginVisitStats( url=f"https://example.com/origin-{i:03d}", visit_type="git", last_successful=utcnow(), last_visit=utcnow(), ) for i in range( page_size + 1 ) # Ensure overflow of the psycopg2.extras.execute_values page_size ] swh_scheduler.origin_visit_stats_upsert(visit_stats) assert set( swh_scheduler.origin_visit_stats_get( [(ovs.url, ovs.visit_type) for ovs in visit_stats] ) ) == set(visit_stats) def test_origin_visit_stats_upsert(self, swh_scheduler) -> None: eventful_date = utcnow() url = "https://github.com/test" visit_stats = OriginVisitStats( url=url, visit_type="git", last_successful=eventful_date, last_visit=eventful_date, ) swh_scheduler.origin_visit_stats_upsert([visit_stats]) swh_scheduler.origin_visit_stats_upsert([visit_stats]) assert swh_scheduler.origin_visit_stats_get([(url, "git")]) == [visit_stats] assert swh_scheduler.origin_visit_stats_get([(url, "svn")]) == [] new_visit_date = utcnow() visit_stats = OriginVisitStats( url=url, visit_type="git", last_successful=None, last_visit=new_visit_date, ) swh_scheduler.origin_visit_stats_upsert([visit_stats]) uneventful_visits = swh_scheduler.origin_visit_stats_get([(url, "git")]) expected_visit_stats = OriginVisitStats( url=url, visit_type="git", last_successful=eventful_date, last_visit=new_visit_date, ) assert uneventful_visits == [expected_visit_stats] def test_origin_visit_stats_upsert_with_snapshot(self, swh_scheduler) -> None: eventful_date = utcnow() url = "https://github.com/666/test" visit_stats = OriginVisitStats( url=url, visit_type="git", last_successful=eventful_date, last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"), ) swh_scheduler.origin_visit_stats_upsert([visit_stats]) assert swh_scheduler.origin_visit_stats_get([(url, "git")]) == [visit_stats] assert swh_scheduler.origin_visit_stats_get([(url, "svn")]) == [] def test_origin_visit_stats_upsert_batch(self, swh_scheduler) -> None: """Batch upsert is ok""" visit_stats = [ OriginVisitStats( url="foo", visit_type="git", last_successful=utcnow(), last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"), ), OriginVisitStats( url="bar", visit_type="git", last_visit=utcnow(), last_snapshot=hash_to_bytes("fffcc0710eb6cf9efd5b920a8453e1e07157bfff"), ), ] swh_scheduler.origin_visit_stats_upsert(visit_stats) for visit_stat in swh_scheduler.origin_visit_stats_get( [(vs.url, vs.visit_type) for vs in visit_stats] ): assert visit_stat is not None def test_origin_visit_stats_upsert_cardinality_failing(self, swh_scheduler) -> None: """Batch upsert does not support altering multiple times the same origin-visit-status """ with pytest.raises(SchedulerException, match="CardinalityViolation"): swh_scheduler.origin_visit_stats_upsert( [ OriginVisitStats( url="foo", visit_type="git", last_successful=None, last_visit=utcnow(), ), OriginVisitStats( url="foo", visit_type="git", last_successful=utcnow(), last_visit=None, ), ] ) def test_visit_scheduler_queue_position( self, swh_scheduler, listed_origins ) -> None: result = swh_scheduler.visit_scheduler_queue_position_get() assert result == {} expected_result = {} visit_types = set() for origin in listed_origins: visit_type = origin.visit_type if visit_type in visit_types: continue visit_types.add(visit_type) position = 42 swh_scheduler.visit_scheduler_queue_position_set(visit_type, position) expected_result[visit_type] = position result = swh_scheduler.visit_scheduler_queue_position_get() assert result == expected_result def test_metrics_origins_known(self, swh_scheduler, listed_origins): swh_scheduler.record_listed_origins(listed_origins) ret = swh_scheduler.update_metrics() assert sum(metric.origins_known for metric in ret) == len(listed_origins) def test_metrics_origins_enabled(self, swh_scheduler, listed_origins): swh_scheduler.record_listed_origins(listed_origins) disabled_origin = attr.evolve(listed_origins[0], enabled=False) swh_scheduler.record_listed_origins([disabled_origin]) ret = swh_scheduler.update_metrics(lister_id=disabled_origin.lister_id) for metric in ret: if metric.visit_type == disabled_origin.visit_type: # We disabled one of these origins assert metric.origins_known - metric.origins_enabled == 1 else: # But these are still all enabled assert metric.origins_known == metric.origins_enabled def test_metrics_origins_never_visited(self, swh_scheduler, listed_origins): swh_scheduler.record_listed_origins(listed_origins) # Pretend that we've recorded a visit on one origin visited_origin = listed_origins[0] swh_scheduler.origin_visit_stats_upsert( [ OriginVisitStats( url=visited_origin.url, visit_type=visited_origin.visit_type, last_successful=utcnow(), last_snapshot=hash_to_bytes( "d81cc0710eb6cf9efd5b920a8453e1e07157b6cd" ), ), ] ) ret = swh_scheduler.update_metrics(lister_id=visited_origin.lister_id) for metric in ret: if metric.visit_type == visited_origin.visit_type: # We visited one of these origins assert metric.origins_known - metric.origins_never_visited == 1 else: # But none of these have been visited assert metric.origins_known == metric.origins_never_visited def test_metrics_origins_with_pending_changes(self, swh_scheduler, listed_origins): swh_scheduler.record_listed_origins(listed_origins) # Pretend that we've recorded a visit on one origin, in the past with # respect to the "last update" time for the origin visited_origin = listed_origins[0] assert visited_origin.last_update is not None swh_scheduler.origin_visit_stats_upsert( [ OriginVisitStats( url=visited_origin.url, visit_type=visited_origin.visit_type, last_successful=visited_origin.last_update - timedelta(days=1), last_snapshot=hash_to_bytes( "d81cc0710eb6cf9efd5b920a8453e1e07157b6cd" ), ), ] ) ret = swh_scheduler.update_metrics(lister_id=visited_origin.lister_id) for metric in ret: if metric.visit_type == visited_origin.visit_type: # We visited one of these origins, in the past assert metric.origins_with_pending_changes == 1 else: # But none of these have been visited assert metric.origins_with_pending_changes == 0 def test_update_metrics_explicit_lister(self, swh_scheduler, listed_origins): swh_scheduler.record_listed_origins(listed_origins) fake_uuid = uuid.uuid4() assert all(fake_uuid != origin.lister_id for origin in listed_origins) ret = swh_scheduler.update_metrics(lister_id=fake_uuid) assert len(ret) == 0 def test_update_metrics_explicit_timestamp(self, swh_scheduler, listed_origins): swh_scheduler.record_listed_origins(listed_origins) ts = datetime.datetime(2020, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc) ret = swh_scheduler.update_metrics(timestamp=ts) assert all(metric.last_update == ts for metric in ret) def test_update_metrics_twice(self, swh_scheduler, listed_origins): swh_scheduler.record_listed_origins(listed_origins) ts = utcnow() ret = swh_scheduler.update_metrics(timestamp=ts) assert all(metric.last_update == ts for metric in ret) second_ts = ts + timedelta(seconds=1) ret = swh_scheduler.update_metrics(timestamp=second_ts) assert all(metric.last_update == second_ts for metric in ret) def test_get_metrics(self, swh_scheduler, listed_origins): swh_scheduler.record_listed_origins(listed_origins) updated = swh_scheduler.update_metrics() retrieved = swh_scheduler.get_metrics() assert_metrics_equal(updated, retrieved) def test_get_metrics_by_lister(self, swh_scheduler, listed_origins): lister_id = listed_origins[0].lister_id assert lister_id is not None swh_scheduler.record_listed_origins(listed_origins) updated = swh_scheduler.update_metrics() retrieved = swh_scheduler.get_metrics(lister_id=lister_id) assert len(retrieved) > 0 assert_metrics_equal( [metric for metric in updated if metric.lister_id == lister_id], retrieved ) def test_get_metrics_by_visit_type(self, swh_scheduler, listed_origins): visit_type = listed_origins[0].visit_type assert visit_type is not None swh_scheduler.record_listed_origins(listed_origins) updated = swh_scheduler.update_metrics() retrieved = swh_scheduler.get_metrics(visit_type=visit_type) assert len(retrieved) > 0 assert_metrics_equal( [metric for metric in updated if metric.visit_type == visit_type], retrieved )