diff --git a/conftest.py b/conftest.py index 57b3d14..de31662 100644 --- a/conftest.py +++ b/conftest.py @@ -1,11 +1,19 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + from hypothesis import settings # define tests profile. Full documentation is at: # https://hypothesis.readthedocs.io/en/latest/settings.html#settings-profiles settings.register_profile("fast", max_examples=5, deadline=5000) settings.register_profile("slow", max_examples=20, deadline=5000) # Ignore the following modules because wsgi module fails as no # configuration file is found (--doctest-modules forces the module # loading) collect_ignore = ["swh/indexer/storage/api/wsgi.py"] + +# we use the swh_scheduler fixture +pytest_plugins = ["swh.scheduler.pytest_plugin"] diff --git a/pytest.ini b/pytest.ini index afa4cf3..4b8d2d3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,3 @@ [pytest] +addopts = -p no:pytest_swh_scheduler norecursedirs = docs diff --git a/requirements-test.txt b/requirements-test.txt index 68bb694..ac0c1f0 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,5 @@ confluent-kafka pytest -pytest-postgresql hypothesis>=3.11.0 -swh.storage>= 0.0.178 +swh.scheduler[testing] >= 0.5.0 +swh.storage[testing] >= 0.10.0 diff --git a/swh/indexer/tests/conftest.py b/swh/indexer/tests/conftest.py index fb25abd..fa85d75 100644 --- a/swh/indexer/tests/conftest.py +++ b/swh/indexer/tests/conftest.py @@ -1,86 +1,77 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import timedelta from unittest.mock import patch import pytest from swh.objstorage import get_objstorage -from swh.scheduler.tests.conftest import * # noqa from swh.storage import get_storage from swh.indexer.storage import get_indexer_storage from .utils import fill_storage, fill_obj_storage TASK_NAMES = ["revision_intrinsic_metadata", "origin_intrinsic_metadata"] storage_config = {"cls": "pipeline", "steps": [{"cls": "validate"}, {"cls": "memory"},]} @pytest.fixture def indexer_scheduler(swh_scheduler): for taskname in TASK_NAMES: swh_scheduler.create_task_type( { "type": taskname, "description": "The {} indexer testing task".format(taskname), "backend_name": "swh.indexer.tests.tasks.{}".format(taskname), "default_interval": timedelta(days=1), "min_interval": timedelta(hours=6), "max_interval": timedelta(days=12), "num_retries": 3, } ) return swh_scheduler @pytest.fixture def idx_storage(): """An instance of in-memory indexer storage that gets injected into all indexers classes. """ idx_storage = get_indexer_storage("memory", {}) with patch("swh.indexer.storage.in_memory.IndexerStorage") as idx_storage_mock: idx_storage_mock.return_value = idx_storage yield idx_storage @pytest.fixture def storage(): """An instance of in-memory storage that gets injected into all indexers classes. """ storage = get_storage(**storage_config) fill_storage(storage) with patch("swh.storage.in_memory.InMemoryStorage") as storage_mock: storage_mock.return_value = storage yield storage @pytest.fixture def obj_storage(): """An instance of in-memory objstorage that gets injected into all indexers classes. """ objstorage = get_objstorage("memory", {}) fill_obj_storage(objstorage) with patch.dict( "swh.objstorage.factory._STORAGE_CLASSES", {"memory": lambda: objstorage} ): yield objstorage - - -@pytest.fixture(scope="session") # type: ignore # expected redefinition -def celery_includes(): - return [ - "swh.indexer.tests.tasks", - "swh.indexer.tasks", - ] diff --git a/swh/indexer/tests/storage/conftest.py b/swh/indexer/tests/storage/conftest.py index e2df26c..a67b2dc 100644 --- a/swh/indexer/tests/storage/conftest.py +++ b/swh/indexer/tests/storage/conftest.py @@ -1,73 +1,73 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from os.path import join import pytest from . import SQL_DIR -from swh.storage.tests.conftest import postgresql_fact +from swh.storage.pytest_plugin import postgresql_fact from swh.indexer.storage import get_indexer_storage from swh.model.hashutil import hash_to_bytes from .generate_data_test import MIMETYPE_OBJECTS, FOSSOLOGY_LICENSES, TOOLS DUMP_FILES = join(SQL_DIR, "*.sql") class DataObj(dict): def __getattr__(self, key): return self.__getitem__(key) def __setattr__(self, key, value): return self.__setitem__(key, value) @pytest.fixture def swh_indexer_storage_with_data(swh_indexer_storage): data = DataObj() tools = { tool["tool_name"]: { "id": tool["id"], "name": tool["tool_name"], "version": tool["tool_version"], "configuration": tool["tool_configuration"], } for tool in swh_indexer_storage.indexer_configuration_add(TOOLS) } data.tools = tools data.sha1_1 = hash_to_bytes("34973274ccef6ab4dfaaf86599792fa9c3fe4689") data.sha1_2 = hash_to_bytes("61c2b3a30496d329e21af70dd2d7e097046d07b7") data.revision_id_1 = hash_to_bytes("7026b7c1a2af56521e951c01ed20f255fa054238") data.revision_id_2 = hash_to_bytes("7026b7c1a2af56521e9587659012345678904321") data.revision_id_3 = hash_to_bytes("7026b7c1a2af56521e9587659012345678904320") data.origin_url_1 = "file:///dev/0/zero" # 44434341 data.origin_url_2 = "file:///dev/1/one" # 44434342 data.origin_url_3 = "file:///dev/2/two" # 54974445 data.mimetypes = [ {**mimetype_obj, "indexer_configuration_id": tools["file"]["id"]} for mimetype_obj in MIMETYPE_OBJECTS ] swh_indexer_storage.content_mimetype_add(data.mimetypes) data.fossology_licenses = [ {**fossology_obj, "indexer_configuration_id": tools["nomos"]["id"]} for fossology_obj in FOSSOLOGY_LICENSES ] swh_indexer_storage._test_data = data return (swh_indexer_storage, data) swh_indexer_storage_postgresql = postgresql_fact( "postgresql_proc", dump_files=DUMP_FILES ) @pytest.fixture def swh_indexer_storage(swh_indexer_storage_postgresql): storage_config = { "cls": "local", "args": {"db": swh_indexer_storage_postgresql.dsn,}, } return get_indexer_storage(**storage_config) diff --git a/tox.ini b/tox.ini index 6bc37bb..1e22dc3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,39 +1,41 @@ [tox] envlist=black,flake8,mypy,py3 [testenv] extras = testing deps = pytest-cov pifpaf + swh-scheduler[testing] >= 0.5.0 + swh-storage[testing] >= 0.10.0 dev: pdbpp commands = pifpaf run postgresql -- pytest --doctest-modules \ !slow: --hypothesis-profile=fast \ slow: --hypothesis-profile=slow \ {envsitepackagesdir}/swh/indexer \ --cov={envsitepackagesdir}/swh/indexer \ --cov-branch {posargs} [testenv:black] skip_install = true deps = black commands = {envpython} -m black --check swh [testenv:flake8] skip_install = true deps = flake8 commands = {envpython} -m flake8 [testenv:mypy] extras = testing deps = mypy commands = mypy swh