diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,2 +1,2 @@ -swh.core[db,http] >= 0.14.0 +swh.core[db,http] >= 2 swh.storage >= 0.11.1 diff --git a/requirements-test.txt b/requirements-test.txt --- a/requirements-test.txt +++ b/requirements-test.txt @@ -8,4 +8,5 @@ types-flask types-pyyaml types-requests +types-toml types-Deprecated diff --git a/swh/scheduler/__init__.py b/swh/scheduler/__init__.py --- a/swh/scheduler/__init__.py +++ b/swh/scheduler/__init__.py @@ -24,8 +24,10 @@ BACKEND_TYPES: Dict[str, str] = { - "local": ".backend.SchedulerBackend", + "postgresql": ".backend.SchedulerBackend", "remote": ".api.client.RemoteScheduler", + # deprecated + "local": ".backend.SchedulerBackend", } @@ -66,3 +68,6 @@ module = import_module(module_path, package=__package__) BackendClass = getattr(module, class_name) return BackendClass(**kwargs) + + +get_datastore = get_scheduler diff --git a/swh/scheduler/backend.py b/swh/scheduler/backend.py --- a/swh/scheduler/backend.py +++ b/swh/scheduler/backend.py @@ -55,6 +55,8 @@ """ + current_version = 33 + def __init__(self, db, min_pool_conns=1, max_pool_conns=10): """ Args: @@ -73,6 +75,9 @@ ) self._db = None + def get_current_version(self): + return self.current_version + def get_db(self): if self._db: return self._db diff --git a/swh/scheduler/pytest_plugin.py b/swh/scheduler/pytest_plugin.py --- a/swh/scheduler/pytest_plugin.py +++ b/swh/scheduler/pytest_plugin.py @@ -4,32 +4,38 @@ # See top-level LICENSE file for more information from datetime import timedelta -import os +from functools import partial from celery.contrib.testing import worker from celery.contrib.testing.app import TestApp, setup_default_app import pkg_resources import pytest +from pytest_postgresql import factories -from swh.core.db.pytest_plugin import postgresql_fact -import swh.scheduler +from swh.core.db.pytest_plugin import initialize_database_for_module, postgresql_fact from swh.scheduler import get_scheduler - -SQL_DIR = os.path.join(os.path.dirname(swh.scheduler.__file__), "sql") +from swh.scheduler.backend import SchedulerBackend # celery tasks for testing purpose; tasks themselves should be # in swh/scheduler/tests/tasks.py TASK_NAMES = ["ping", "multiping", "add", "error", "echo"] -postgresql_scheduler = postgresql_fact( - "postgresql_proc", +scheduler_postgresql_proc = factories.postgresql_proc( dbname="scheduler", - dump_files=os.path.join(SQL_DIR, "*.sql"), - no_truncate_tables={"dbversion", "priority_ratio"}, + load=[ + partial( + initialize_database_for_module, + modname="scheduler", + version=SchedulerBackend.current_version, + ) + ], ) +postgresql_scheduler = postgresql_fact("scheduler_postgresql_proc") + + @pytest.fixture def swh_scheduler_config(request, postgresql_scheduler): return { diff --git a/swh/scheduler/sql/30-schema.sql b/swh/scheduler/sql/30-schema.sql --- a/swh/scheduler/sql/30-schema.sql +++ b/swh/scheduler/sql/30-schema.sql @@ -1,18 +1,3 @@ -create table dbversion -( - version int primary key, - release timestamptz not null, - description text not null -); - -comment on table dbversion is 'Schema update tracking'; -comment on column dbversion.version is 'SQL schema version'; -comment on column dbversion.release is 'Version deployment timestamp'; -comment on column dbversion.description is 'Version description'; - -insert into dbversion (version, release, description) - values (33, now(), 'Work In Progress'); - create table task_type ( type text primary key, description text not null, diff --git a/sql/updates/02.sql b/swh/scheduler/sql/upgrades/02.sql rename from sql/updates/02.sql rename to swh/scheduler/sql/upgrades/02.sql diff --git a/sql/updates/03.sql b/swh/scheduler/sql/upgrades/03.sql rename from sql/updates/03.sql rename to swh/scheduler/sql/upgrades/03.sql diff --git a/sql/updates/04.sql b/swh/scheduler/sql/upgrades/04.sql rename from sql/updates/04.sql rename to swh/scheduler/sql/upgrades/04.sql diff --git a/sql/updates/05.sql b/swh/scheduler/sql/upgrades/05.sql rename from sql/updates/05.sql rename to swh/scheduler/sql/upgrades/05.sql diff --git a/sql/updates/06.sql b/swh/scheduler/sql/upgrades/06.sql rename from sql/updates/06.sql rename to swh/scheduler/sql/upgrades/06.sql diff --git a/sql/updates/07.sql b/swh/scheduler/sql/upgrades/07.sql rename from sql/updates/07.sql rename to swh/scheduler/sql/upgrades/07.sql diff --git a/sql/updates/08.sql b/swh/scheduler/sql/upgrades/08.sql rename from sql/updates/08.sql rename to swh/scheduler/sql/upgrades/08.sql diff --git a/sql/updates/09.sql b/swh/scheduler/sql/upgrades/09.sql rename from sql/updates/09.sql rename to swh/scheduler/sql/upgrades/09.sql diff --git a/sql/updates/10.sql b/swh/scheduler/sql/upgrades/10.sql rename from sql/updates/10.sql rename to swh/scheduler/sql/upgrades/10.sql diff --git a/sql/updates/11.sql b/swh/scheduler/sql/upgrades/11.sql rename from sql/updates/11.sql rename to swh/scheduler/sql/upgrades/11.sql diff --git a/sql/updates/12.sql b/swh/scheduler/sql/upgrades/12.sql rename from sql/updates/12.sql rename to swh/scheduler/sql/upgrades/12.sql diff --git a/sql/updates/13.sql b/swh/scheduler/sql/upgrades/13.sql rename from sql/updates/13.sql rename to swh/scheduler/sql/upgrades/13.sql diff --git a/sql/updates/14.sql b/swh/scheduler/sql/upgrades/14.sql rename from sql/updates/14.sql rename to swh/scheduler/sql/upgrades/14.sql diff --git a/sql/updates/15.sql b/swh/scheduler/sql/upgrades/15.sql rename from sql/updates/15.sql rename to swh/scheduler/sql/upgrades/15.sql diff --git a/sql/updates/16.sql b/swh/scheduler/sql/upgrades/16.sql rename from sql/updates/16.sql rename to swh/scheduler/sql/upgrades/16.sql diff --git a/sql/updates/17.sql b/swh/scheduler/sql/upgrades/17.sql rename from sql/updates/17.sql rename to swh/scheduler/sql/upgrades/17.sql diff --git a/sql/updates/18.sql b/swh/scheduler/sql/upgrades/18.sql rename from sql/updates/18.sql rename to swh/scheduler/sql/upgrades/18.sql diff --git a/sql/updates/19.sql b/swh/scheduler/sql/upgrades/19.sql rename from sql/updates/19.sql rename to swh/scheduler/sql/upgrades/19.sql diff --git a/sql/updates/20.sql b/swh/scheduler/sql/upgrades/20.sql rename from sql/updates/20.sql rename to swh/scheduler/sql/upgrades/20.sql diff --git a/sql/updates/23.sql b/swh/scheduler/sql/upgrades/23.sql rename from sql/updates/23.sql rename to swh/scheduler/sql/upgrades/23.sql diff --git a/sql/updates/24.sql b/swh/scheduler/sql/upgrades/24.sql rename from sql/updates/24.sql rename to swh/scheduler/sql/upgrades/24.sql diff --git a/sql/updates/25.sql b/swh/scheduler/sql/upgrades/25.sql rename from sql/updates/25.sql rename to swh/scheduler/sql/upgrades/25.sql diff --git a/sql/updates/26.sql b/swh/scheduler/sql/upgrades/26.sql rename from sql/updates/26.sql rename to swh/scheduler/sql/upgrades/26.sql diff --git a/sql/updates/27.sql b/swh/scheduler/sql/upgrades/27.sql rename from sql/updates/27.sql rename to swh/scheduler/sql/upgrades/27.sql diff --git a/sql/updates/28.sql b/swh/scheduler/sql/upgrades/28.sql rename from sql/updates/28.sql rename to swh/scheduler/sql/upgrades/28.sql diff --git a/sql/updates/29.sql b/swh/scheduler/sql/upgrades/29.sql rename from sql/updates/29.sql rename to swh/scheduler/sql/upgrades/29.sql diff --git a/sql/updates/30-bis.sql b/swh/scheduler/sql/upgrades/30-bis.sql rename from sql/updates/30-bis.sql rename to swh/scheduler/sql/upgrades/30-bis.sql diff --git a/sql/updates/30.sql b/swh/scheduler/sql/upgrades/30.sql rename from sql/updates/30.sql rename to swh/scheduler/sql/upgrades/30.sql diff --git a/sql/updates/31.sql b/swh/scheduler/sql/upgrades/31.sql rename from sql/updates/31.sql rename to swh/scheduler/sql/upgrades/31.sql diff --git a/sql/updates/32.sql b/swh/scheduler/sql/upgrades/32.sql rename from sql/updates/32.sql rename to swh/scheduler/sql/upgrades/32.sql diff --git a/sql/updates/33.sql b/swh/scheduler/sql/upgrades/33.sql rename from sql/updates/33.sql rename to swh/scheduler/sql/upgrades/33.sql