diff --git a/mypy.ini b/mypy.ini index 496e8d9..f4d73e8 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,27 +1,30 @@ [mypy] namespace_packages = True warn_unused_ignores = True # 3rd party libraries without stubs (yet) [mypy-arrow.*] ignore_missing_imports = True [mypy-celery.*] ignore_missing_imports = True [mypy-elasticsearch.*] ignore_missing_imports = True [mypy-kombu.*] ignore_missing_imports = True [mypy-pkg_resources.*] ignore_missing_imports = True [mypy-psycopg2.*] ignore_missing_imports = True [mypy-pytest.*] ignore_missing_imports = True + +[mypy-pytest_postgresql.*] +ignore_missing_imports = True diff --git a/requirements-test.txt b/requirements-test.txt index 58dc6ac..539ca68 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,5 @@ # pytest<4 because of https://github.com/pytest-dev/pytest/issues/4641 pytest < 4 -pytest-postgresql +pytest-postgresql >= 2.1.0 celery >= 4 hypothesis >= 3.11.0 diff --git a/swh/scheduler/tests/updater/conftest.py b/swh/scheduler/tests/updater/conftest.py index 9cafcb5..87584c5 100644 --- a/swh/scheduler/tests/updater/conftest.py +++ b/swh/scheduler/tests/updater/conftest.py @@ -1,33 +1,68 @@ import pytest import glob import os +from arrow import utcnow # XXX from swh.core.utils import numfile_sortkey as sortkey from swh.scheduler.updater.backend import SchedulerUpdaterBackend from swh.scheduler.tests import SQL_DIR import swh.scheduler.tests.conftest # noqa DUMP_FILES = os.path.join(SQL_DIR, 'updater', '*.sql') @pytest.fixture -def swh_scheduler_updater(request, postgresql_proc, postgresql): +def swh_scheduler_updater(postgresql): config = { - 'db': 'postgresql://{user}@{host}:{port}/{dbname}'.format( - host=postgresql_proc.host, - port=postgresql_proc.port, - user='postgres', - dbname='tests') + 'db': postgresql.dsn, } all_dump_files = sorted(glob.glob(DUMP_FILES), key=sortkey) cursor = postgresql.cursor() for fname in all_dump_files: with open(fname) as fobj: cursor.execute(fobj.read()) postgresql.commit() backend = SchedulerUpdaterBackend(**config) return backend + + +def make_event(event_type, name, origin_type): + return { + 'type': event_type, + 'repo': { + 'name': name, + }, + 'created_at': utcnow(), + 'origin_type': origin_type, + } + + +def make_simple_event(event_type, name, origin_type): + return { + 'type': event_type, + 'url': 'https://fakeurl/%s' % name, + 'origin_type': origin_type, + 'created_at': utcnow(), + } + + +def make_events(events): + for event_type, repo_name, origin_type in events: + yield make_event(event_type, repo_name, origin_type) + + +def make_incomplete_event(event_type, name, origin_type, + missing_data_key): + event = make_event(event_type, name, origin_type) + del event[missing_data_key] + return event + + +def make_incomplete_events(events): + for event_type, repo_name, origin_type, missing_data_key in events: + yield make_incomplete_event(event_type, repo_name, + origin_type, missing_data_key) diff --git a/swh/scheduler/tests/updater/test_writer.py b/swh/scheduler/tests/updater/test_writer.py index 2e084e9..825f4a2 100644 --- a/swh/scheduler/tests/updater/test_writer.py +++ b/swh/scheduler/tests/updater/test_writer.py @@ -1,153 +1,152 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os -import unittest from glob import glob import pytest +from pytest_postgresql.factories import postgresql as pg_fixture_factory +from os.path import join from swh.core.utils import numfile_sortkey as sortkey -from swh.core.db.tests.db_testing import DbTestFixture from swh.scheduler.tests import SQL_DIR from swh.scheduler.updater.events import LISTENED_EVENTS, SWHEvent from swh.scheduler.updater.writer import UpdaterWriter -from . import UpdaterTestUtil - - -@pytest.mark.db -class CommonSchedulerTest(DbTestFixture): - TEST_SCHED_DB = 'softwareheritage-scheduler-test' - TEST_SCHED_DUMP = os.path.join(SQL_DIR, '*.sql') - - TEST_SCHED_UPDATER_DB = 'softwareheritage-scheduler-updater-test' - TEST_SCHED_UPDATER_DUMP = os.path.join(SQL_DIR, 'updater', '*.sql') - - @classmethod - def setUpClass(cls): - cls.add_db(cls.TEST_SCHED_DB, - [(sqlfn, 'psql') for sqlfn in - sorted(glob(cls.TEST_SCHED_DUMP), key=sortkey)]) - cls.add_db(cls.TEST_SCHED_UPDATER_DB, - [(sqlfn, 'psql') for sqlfn in - sorted(glob(cls.TEST_SCHED_UPDATER_DUMP), key=sortkey)]) - super().setUpClass() - - def tearDown(self): - self.reset_db_tables(self.TEST_SCHED_UPDATER_DB) - self.reset_db_tables(self.TEST_SCHED_DB, - excluded=['task_type', 'priority_ratio']) - super().tearDown() - - -class UpdaterWriterTest(UpdaterTestUtil, CommonSchedulerTest, - unittest.TestCase): - def setUp(self): - super().setUp() - - config = { - 'scheduler': { - 'cls': 'local', - 'args': { - 'db': 'dbname=softwareheritage-scheduler-test', - }, - }, - 'scheduler_updater': { - 'cls': 'local', - 'args': { - 'db': - 'dbname=softwareheritage-scheduler-updater-test', - 'cache_read_limit': 5, - }, +from .conftest import make_simple_event + + +pg_scheduler = pg_fixture_factory('postgresql_proc', 'scheduler') +pg_updater = pg_fixture_factory('postgresql_proc', 'updater') + + +def pg_sched_fact(dbname, sqldir): + @pytest.fixture + def pg_scheduler_db(request): + pg = request.getfixturevalue('pg_%s' % dbname) + dump_files = sorted(glob(os.path.join(sqldir, '*.sql')), + key=sortkey) + with pg.cursor() as cur: + for fname in dump_files: + with open(fname) as fobj: + sql = fobj.read().replace('concurrently', '') + cur.execute(sql) + pg.commit() + yield pg + + return pg_scheduler_db + + +scheduler_db = pg_sched_fact('scheduler', SQL_DIR) +updater_db = pg_sched_fact('updater', join(SQL_DIR, 'updater')) + + +@pytest.fixture +def swh_updater_writer(scheduler_db, updater_db): + config = { + 'scheduler': { + 'cls': 'local', + 'args': { + 'db': scheduler_db.dsn, }, - 'updater_writer': { - 'pause': 0.1, - 'verbose': False, + }, + 'scheduler_updater': { + 'cls': 'local', + 'args': { + 'db': updater_db.dsn, + 'cache_read_limit': 5, }, - } - self.writer = UpdaterWriter(**config) - self.scheduler_backend = self.writer.scheduler_backend - self.scheduler_updater_backend = self.writer.scheduler_updater_backend + }, + 'updater_writer': { + 'pause': 0.1, + 'verbose': False, + }, + } + return UpdaterWriter(**config) + + +def test_run_ko(swh_updater_writer): + """Only git tasks are supported for now, other types are dismissed. + + """ + scheduler = swh_updater_writer.scheduler_backend + updater = swh_updater_writer.scheduler_updater_backend - def test_run_ko(self): - """Only git tasks are supported for now, other types are dismissed. + ready_events = [ + SWHEvent( + make_simple_event(event_type, 'origin-%s' % i, + 'svn')) + for i, event_type in enumerate(LISTENED_EVENTS) + ] - """ - ready_events = [ - SWHEvent( - self._make_simple_event(event_type, 'origin-%s' % i, - 'svn')) - for i, event_type in enumerate(LISTENED_EVENTS) - ] + updater.cache_put(ready_events) + list(updater.cache_read()) - expected_length = len(ready_events) + r = scheduler.peek_ready_tasks('load-git') - self.scheduler_updater_backend.cache_put(ready_events) - data = list(self.scheduler_updater_backend.cache_read()) - self.assertEqual(len(data), expected_length) + # first read on an empty scheduling db results with nothing in it + assert not r - r = self.scheduler_backend.peek_ready_tasks('load-git') + # Read from cache to scheduler db + swh_updater_writer.run() - # first read on an empty scheduling db results with nothing in it - self.assertEqual(len(r), 0) + r = scheduler.peek_ready_tasks('load-git') - # Read from cache to scheduler db - self.writer.run() + # other reads after writes are still empty since it's not supported + assert not r - r = self.scheduler_backend.peek_ready_tasks('load-git') - # other reads after writes are still empty since it's not supported - self.assertEqual(len(r), 0) +def test_run_ok(swh_updater_writer): + """Only git origin are supported for now - def test_run_ok(self): - """Only git origin are supported for now + """ + scheduler = swh_updater_writer.scheduler_backend + updater = swh_updater_writer.scheduler_updater_backend - """ - ready_events = [ - SWHEvent( - self._make_simple_event(event_type, 'origin-%s' % i, 'git')) - for i, event_type in enumerate(LISTENED_EVENTS) - ] + ready_events = [ + SWHEvent( + make_simple_event(event_type, 'origin-%s' % i, 'git')) + for i, event_type in enumerate(LISTENED_EVENTS) + ] - expected_length = len(ready_events) + expected_length = len(ready_events) - self.scheduler_updater_backend.cache_put(ready_events) + updater.cache_put(ready_events) - data = list(self.scheduler_updater_backend.cache_read()) - self.assertEqual(len(data), expected_length) + data = list(updater.cache_read()) + assert len(data) == expected_length - r = self.scheduler_backend.peek_ready_tasks('load-git') + r = scheduler.peek_ready_tasks('load-git') - # first read on an empty scheduling db results with nothing in it - self.assertEqual(len(r), 0) + # first read on an empty scheduling db results with nothing in it + assert not r - # Read from cache to scheduler db - self.writer.run() + # Read from cache to scheduler db + swh_updater_writer.run() - # now, we should have scheduling task ready - r = self.scheduler_backend.peek_ready_tasks('load-git') + # now, we should have scheduling task ready + r = scheduler.peek_ready_tasks('load-git') - self.assertEqual(len(r), expected_length) + assert len(r) == expected_length - # Check the task has been scheduled - for t in r: - self.assertEqual(t['type'], 'load-git') - self.assertEqual(t['priority'], 'normal') - self.assertEqual(t['policy'], 'oneshot') - self.assertEqual(t['status'], 'next_run_not_scheduled') + # Check the task has been scheduled + for t in r: + assert t['type'] == 'load-git' + assert t['priority'] == 'normal' + assert t['policy'] == 'oneshot' + assert t['status'] == 'next_run_not_scheduled' - # writer has nothing to do now - self.writer.run() + # writer has nothing to do now + swh_updater_writer.run() - # so no more data in cache - data = list(self.scheduler_updater_backend.cache_read()) + # so no more data in cache + data = list(updater.cache_read()) - self.assertEqual(len(data), 0) + assert not data - # provided, no runner is ran, still the same amount of scheduling tasks - r = self.scheduler_backend.peek_ready_tasks('load-git') + # provided, no runner is ran, still the same amount of scheduling tasks + r = scheduler.peek_ready_tasks('load-git') - self.assertEqual(len(r), expected_length) + assert len(r) == expected_length