Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7124292
D1027.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
10 KB
Subscribers
None
D1027.diff
View Options
diff --git a/swh/scheduler/tests/conftest.py b/swh/scheduler/tests/conftest.py
--- a/swh/scheduler/tests/conftest.py
+++ b/swh/scheduler/tests/conftest.py
@@ -3,6 +3,12 @@
import glob
from datetime import timedelta
+import swh.scheduler.celery_backend.config
+# this import is needed here to enforce creation of the celery current app
+# BEFORE the swh_app fixture is called, otherwise the Celery app instance from
+# celery_backend.config becomes the celery.current_app
+
+
from swh.core.utils import numfile_sortkey as sortkey
from swh.scheduler import get_scheduler
from swh.scheduler.tests import SQL_DIR
@@ -47,7 +53,6 @@
# with the test application.
@pytest.fixture(scope='session')
def swh_app(celery_session_app):
- import swh.scheduler.celery_backend.config
swh.scheduler.celery_backend.config.app = celery_session_app
yield celery_session_app
@@ -55,7 +60,7 @@
@pytest.fixture
def swh_scheduler(request, postgresql_proc, postgresql):
scheduler_config = {
- 'scheduling_db': 'postgresql://{user}@{host}:{port}/{dbname}'.format(
+ 'db': 'postgresql://{user}@{host}:{port}/{dbname}'.format(
host=postgresql_proc.host,
port=postgresql_proc.port,
user='postgres',
diff --git a/swh/scheduler/tests/test_api_client.py b/swh/scheduler/tests/test_api_client.py
--- a/swh/scheduler/tests/test_api_client.py
+++ b/swh/scheduler/tests/test_api_client.py
@@ -25,7 +25,7 @@
'scheduler': {
'cls': 'local',
'args': {
- 'scheduling_db': 'dbname=%s' % self.TEST_DB_NAME,
+ 'db': 'dbname=%s' % self.TEST_DB_NAME,
}
}
}
diff --git a/swh/scheduler/tests/test_scheduler.py b/swh/scheduler/tests/test_scheduler.py
--- a/swh/scheduler/tests/test_scheduler.py
+++ b/swh/scheduler/tests/test_scheduler.py
@@ -21,53 +21,60 @@
from . import SQL_DIR
+TASK_TYPES = {
+ 'git': {
+ 'type': 'update-git',
+ 'description': 'Update a git repository',
+ 'backend_name': 'swh.loader.git.tasks.UpdateGitRepository',
+ 'default_interval': datetime.timedelta(days=64),
+ 'min_interval': datetime.timedelta(hours=12),
+ 'max_interval': datetime.timedelta(days=64),
+ 'backoff_factor': 2,
+ 'max_queue_length': None,
+ 'num_retries': 7,
+ 'retry_delay': datetime.timedelta(hours=2),
+ },
+ 'hg': {
+ 'type': 'update-hg',
+ 'description': 'Update a mercurial repository',
+ 'backend_name': 'swh.loader.mercurial.tasks.UpdateHgRepository',
+ 'default_interval': datetime.timedelta(days=64),
+ 'min_interval': datetime.timedelta(hours=12),
+ 'max_interval': datetime.timedelta(days=64),
+ 'backoff_factor': 2,
+ 'max_queue_length': None,
+ 'num_retries': 7,
+ 'retry_delay': datetime.timedelta(hours=2),
+ },
+}
+
+TEMPLATES = {
+ 'git': {
+ 'type': 'update-git',
+ 'arguments': {
+ 'args': [],
+ 'kwargs': {},
+ },
+ 'next_run': None,
+ },
+ 'hg': {
+ 'type': 'update-hg',
+ 'arguments': {
+ 'args': [],
+ 'kwargs': {},
+ },
+ 'next_run': None,
+ 'policy': 'oneshot',
+ }
+}
+
+
@pytest.mark.db
class CommonSchedulerTest(SingleDbTestFixture):
TEST_DB_NAME = 'softwareheritage-scheduler-test'
TEST_DB_DUMP = os.path.join(SQL_DIR, '*.sql')
- def setUp(self):
- super().setUp()
-
- tt = {
- 'type': 'update-git',
- 'description': 'Update a git repository',
- 'backend_name': 'swh.loader.git.tasks.UpdateGitRepository',
- 'default_interval': datetime.timedelta(days=64),
- 'min_interval': datetime.timedelta(hours=12),
- 'max_interval': datetime.timedelta(days=64),
- 'backoff_factor': 2,
- 'max_queue_length': None,
- 'num_retries': 7,
- 'retry_delay': datetime.timedelta(hours=2),
- }
- tt2 = tt.copy()
- tt2['type'] = 'update-hg'
- tt2['description'] = 'Update a mercurial repository'
- tt2['backend_name'] = 'swh.loader.mercurial.tasks.UpdateHgRepository'
- tt2['max_queue_length'] = 42
- tt2['num_retries'] = None
- tt2['retry_delay'] = None
-
- self.task_types = {
- tt['type']: tt,
- tt2['type']: tt2,
- }
-
- self.task1_template = t1_template = {
- 'type': tt['type'],
- 'arguments': {
- 'args': [],
- 'kwargs': {},
- },
- 'next_run': None,
- }
- self.task2_template = t2_template = copy.deepcopy(t1_template)
- t2_template['type'] = tt2['type']
- t2_template['policy'] = 'oneshot'
-
def tearDown(self):
- self.backend.close_connection()
self.empty_tables()
super().tearDown()
@@ -85,18 +92,20 @@
self.conn.commit()
def test_add_task_type(self):
- tt, tt2 = self.task_types.values()
+ tt = TASK_TYPES['git']
self.backend.create_task_type(tt)
self.assertEqual(tt, self.backend.get_task_type(tt['type']))
with self.assertRaisesRegex(psycopg2.IntegrityError,
r'\(type\)=\(%s\)' % tt['type']):
self.backend.create_task_type(tt)
+
+ tt2 = TASK_TYPES['hg']
self.backend.create_task_type(tt2)
self.assertEqual(tt, self.backend.get_task_type(tt['type']))
self.assertEqual(tt2, self.backend.get_task_type(tt2['type']))
def test_get_task_types(self):
- tt, tt2 = self.task_types.values()
+ tt, tt2 = TASK_TYPES['git'], TASK_TYPES['hg']
self.backend.create_task_type(tt)
self.backend.create_task_type(tt2)
self.assertCountEqual([tt2, tt], self.backend.get_task_types())
@@ -143,17 +152,18 @@
return tasks
def _create_task_types(self):
- for tt in self.task_types.values():
+ for tt in TASK_TYPES.values():
self.backend.create_task_type(tt)
def test_create_tasks(self):
priority_ratio = self._priority_ratio()
self._create_task_types()
num_tasks_priority = 100
- tasks_1 = self._tasks_from_template(self.task1_template, utcnow(), 100)
+ tasks_1 = self._tasks_from_template(
+ TEMPLATES['git'], utcnow(), 100)
tasks_2 = self._tasks_from_template(
- self.task2_template, utcnow(), 100,
- num_tasks_priority, priorities=priority_ratio)
+ TEMPLATES['hg'], utcnow(), 100,
+ num_tasks_priority, priorities=priority_ratio)
tasks = tasks_1 + tasks_2
# tasks are returned only once with their ids
@@ -173,7 +183,7 @@
for task, orig_task in zip(ret, tasks):
task = copy.deepcopy(task)
- task_type = self.task_types[orig_task['type']]
+ task_type = TASK_TYPES[orig_task['type'].split('-')[-1]]
self.assertNotIn(task['id'], ids)
self.assertEqual(task['status'], 'next_run_not_scheduled')
self.assertEqual(task['current_interval'],
@@ -205,8 +215,8 @@
def test_peek_ready_tasks_no_priority(self):
self._create_task_types()
t = utcnow()
- task_type = self.task1_template['type']
- tasks = self._tasks_from_template(self.task1_template, t, 100)
+ task_type = TEMPLATES['git']['type']
+ tasks = self._tasks_from_template(TEMPLATES['git'], t, 100)
random.shuffle(tasks)
self.backend.create_tasks(tasks)
@@ -257,12 +267,12 @@
priority_ratio = self._priority_ratio()
self._create_task_types()
t = utcnow()
- task_type = self.task1_template['type']
+ task_type = TEMPLATES['git']['type']
num_tasks_priority = 100
num_tasks_no_priority = 100
# Create tasks with and without priorities
tasks = self._tasks_from_template(
- self.task1_template, t,
+ TEMPLATES['git'], t,
num=num_tasks_no_priority,
num_priority=num_tasks_priority,
priorities=priority_ratio)
@@ -315,12 +325,12 @@
priority_ratio = self._priority_ratio()
self._create_task_types()
t = utcnow()
- task_type = self.task1_template['type']
+ task_type = TEMPLATES['git']['type']
num_tasks_priority = 100
num_tasks_no_priority = 100
# Create tasks with and without priorities
tasks = self._tasks_from_template(
- self.task1_template, t,
+ TEMPLATES['git'], t,
num=num_tasks_no_priority,
num_priority=num_tasks_priority,
priorities=priority_ratio)
@@ -343,7 +353,7 @@
def test_get_tasks(self):
self._create_task_types()
t = utcnow()
- tasks = self._tasks_from_template(self.task1_template, t, 100)
+ tasks = self._tasks_from_template(TEMPLATES['git'], t, 100)
tasks = self.backend.create_tasks(tasks)
random.shuffle(tasks)
while len(tasks) > 1:
@@ -360,8 +370,8 @@
"""
self._create_task_types()
_time = utcnow()
- recurring = self._tasks_from_template(self.task1_template, _time, 12)
- oneshots = self._tasks_from_template(self.task2_template, _time, 12)
+ recurring = self._tasks_from_template(TEMPLATES['git'], _time, 12)
+ oneshots = self._tasks_from_template(TEMPLATES['hg'], _time, 12)
total_tasks = len(recurring) + len(oneshots)
# simulate scheduling tasks
@@ -434,9 +444,9 @@
self._create_task_types()
_time = utcnow()
recurring = self._tasks_from_template(
- self.task1_template, _time, 12)
+ TEMPLATES['git'], _time, 12)
oneshots = self._tasks_from_template(
- self.task2_template, _time, 12)
+ TEMPLATES['hg'], _time, 12)
total_tasks = len(recurring) + len(oneshots)
pending_tasks = self.backend.create_tasks(recurring + oneshots)
backend_tasks = [{
@@ -470,5 +480,5 @@
class LocalSchedulerTest(CommonSchedulerTest, unittest.TestCase):
def setUp(self):
super().setUp()
- self.config = {'scheduling_db': 'dbname=' + self.TEST_DB_NAME}
+ self.config = {'db': 'dbname=' + self.TEST_DB_NAME}
self.backend = get_scheduler('local', self.config)
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Dec 21 2024, 5:05 AM (11 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3223887
Attached To
D1027: Fix tests for the scheduler and the API
Event Timeline
Log In to Comment