Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/tests/conftest.py
| # Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from datetime import timedelta | from datetime import timedelta | ||||
| import os | import os | ||||
| from os import path | |||||
| from typing import List, Tuple | from typing import List, Tuple | ||||
| from unittest.mock import patch | from unittest.mock import patch | ||||
| import pytest | import pytest | ||||
| import yaml | import yaml | ||||
| from swh.core.db.pytest_plugin import postgresql_fact | |||||
| import swh.indexer | |||||
| from swh.indexer.storage import get_indexer_storage | from swh.indexer.storage import get_indexer_storage | ||||
| from swh.objstorage.factory import get_objstorage | from swh.objstorage.factory import get_objstorage | ||||
| from swh.storage import get_storage | from swh.storage import get_storage | ||||
| from .utils import fill_obj_storage, fill_storage | from .utils import fill_obj_storage, fill_storage | ||||
| TASK_NAMES: List[Tuple[str, str]] = [ | TASK_NAMES: List[Tuple[str, str]] = [ | ||||
| # (scheduler-task-type, task-class-test-name) | # (scheduler-task-type, task-class-test-name) | ||||
| ("index-revision-metadata", "revision_intrinsic_metadata"), | ("index-revision-metadata", "revision_intrinsic_metadata"), | ||||
| ("index-origin-metadata", "origin_intrinsic_metadata"), | ("index-origin-metadata", "origin_intrinsic_metadata"), | ||||
| ] | ] | ||||
| SQL_FILES = path.join(path.dirname(swh.indexer.__file__), "sql", "*.sql") | |||||
| idx_storage_postgresql = postgresql_fact( | |||||
| "postgresql_proc", db_name="indexer_storage", dump_files=SQL_FILES, | |||||
| ) | |||||
| @pytest.fixture | @pytest.fixture | ||||
| def indexer_scheduler(swh_scheduler): | def indexer_scheduler(swh_scheduler): | ||||
| # Insert the expected task types within the scheduler | # Insert the expected task types within the scheduler | ||||
| for task_name, task_class_name in TASK_NAMES: | for task_name, task_class_name in TASK_NAMES: | ||||
| swh_scheduler.create_task_type( | swh_scheduler.create_task_type( | ||||
| { | { | ||||
| "type": task_name, | "type": task_name, | ||||
| "description": f"The {task_class_name} indexer testing task", | "description": f"The {task_class_name} indexer testing task", | ||||
| "backend_name": f"swh.indexer.tests.tasks.{task_class_name}", | "backend_name": f"swh.indexer.tests.tasks.{task_class_name}", | ||||
| "default_interval": timedelta(days=1), | "default_interval": timedelta(days=1), | ||||
| "min_interval": timedelta(hours=6), | "min_interval": timedelta(hours=6), | ||||
| "max_interval": timedelta(days=12), | "max_interval": timedelta(days=12), | ||||
| "num_retries": 3, | "num_retries": 3, | ||||
| } | } | ||||
| ) | ) | ||||
| return swh_scheduler | return swh_scheduler | ||||
| @pytest.fixture | @pytest.fixture | ||||
| def idx_storage(): | def idx_storage_backend_config(idx_storage_postgresql): | ||||
| """Basic pg storage configuration with no journal collaborator for the indexer | |||||
| storage (to avoid pulling optional dependency on clients of this fixture) | |||||
| """ | |||||
| return { | |||||
| "cls": "local", | |||||
| "db": idx_storage_postgresql.dsn, | |||||
| } | |||||
| @pytest.fixture | |||||
| def swh_indexer_config( | |||||
| swh_storage_backend_config, idx_storage_backend_config, swh_scheduler_config | |||||
| ): | |||||
| return { | |||||
| "storage": swh_storage_backend_config, | |||||
| "objstorage": {"cls": "memory"}, | |||||
| "indexer_storage": idx_storage_backend_config, | |||||
| "scheduler": {"cls": "local", **swh_scheduler_config}, | |||||
| "tools": { | |||||
| "name": "file", | |||||
| "version": "1:5.30-1+deb9u1", | |||||
| "configuration": {"type": "library", "debian-package": "python3-magic"}, | |||||
| }, | |||||
| "compute_checksums": ["blake2b512"], # for rehash indexer | |||||
| } | |||||
| @pytest.fixture | |||||
| def idx_storage(swh_indexer_config): | |||||
| """An instance of in-memory indexer storage that gets injected into all | """An instance of in-memory indexer storage that gets injected into all | ||||
| indexers classes. | indexers classes. | ||||
| """ | """ | ||||
| idx_storage = get_indexer_storage("memory") | idx_storage_config = swh_indexer_config["indexer_storage"] | ||||
| with patch("swh.indexer.storage.in_memory.IndexerStorage") as idx_storage_mock: | return get_indexer_storage(**idx_storage_config) | ||||
| idx_storage_mock.return_value = idx_storage | |||||
| yield idx_storage | |||||
| @pytest.fixture | @pytest.fixture | ||||
| def storage(): | def storage(swh_indexer_config): | ||||
| """An instance of in-memory storage that gets injected into all indexers | """An instance of in-memory storage that gets injected into all indexers | ||||
| classes. | classes. | ||||
| """ | """ | ||||
| storage = get_storage(cls="memory") | storage = get_storage(**swh_indexer_config["storage"]) | ||||
| fill_storage(storage) | fill_storage(storage) | ||||
| with patch("swh.storage.in_memory.InMemoryStorage") as storage_mock: | return storage | ||||
| storage_mock.return_value = storage | |||||
| yield storage | |||||
| @pytest.fixture | @pytest.fixture | ||||
| def obj_storage(): | def obj_storage(swh_indexer_config): | ||||
| """An instance of in-memory objstorage that gets injected into all indexers | """An instance of in-memory objstorage that gets injected into all indexers | ||||
| classes. | classes. | ||||
| """ | """ | ||||
| objstorage = get_objstorage("memory") | objstorage = get_objstorage(**swh_indexer_config["objstorage"]) | ||||
| fill_obj_storage(objstorage) | fill_obj_storage(objstorage) | ||||
| with patch.dict( | with patch.dict( | ||||
| "swh.objstorage.factory._STORAGE_CLASSES", {"memory": lambda: objstorage} | "swh.objstorage.factory._STORAGE_CLASSES", {"memory": lambda: objstorage} | ||||
| ): | ): | ||||
| yield objstorage | yield objstorage | ||||
| @pytest.fixture | @pytest.fixture | ||||
| def swh_indexer_config(): | |||||
| return { | |||||
| "storage": {"cls": "memory"}, | |||||
| "objstorage": {"cls": "memory"}, | |||||
| "indexer_storage": {"cls": "memory"}, | |||||
| "tools": { | |||||
| "name": "file", | |||||
| "version": "1:5.30-1+deb9u1", | |||||
| "configuration": {"type": "library", "debian-package": "python3-magic"}, | |||||
| }, | |||||
| "compute_checksums": ["blake2b512"], # for rehash indexer | |||||
| } | |||||
| @pytest.fixture | |||||
| def swh_config(swh_indexer_config, monkeypatch, tmp_path): | def swh_config(swh_indexer_config, monkeypatch, tmp_path): | ||||
| conffile = os.path.join(str(tmp_path), "indexer.yml") | conffile = os.path.join(str(tmp_path), "indexer.yml") | ||||
| with open(conffile, "w") as f: | with open(conffile, "w") as f: | ||||
| f.write(yaml.dump(swh_indexer_config)) | f.write(yaml.dump(swh_indexer_config)) | ||||
| monkeypatch.setenv("SWH_CONFIG_FILENAME", conffile) | monkeypatch.setenv("SWH_CONFIG_FILENAME", conffile) | ||||
| return conffile | return conffile | ||||