diff --git a/conftest.py b/conftest.py index de31662..d3cc5fd 100644 --- a/conftest.py +++ b/conftest.py @@ -1,19 +1,28 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import pytest + from hypothesis import settings # define tests profile. Full documentation is at: # https://hypothesis.readthedocs.io/en/latest/settings.html#settings-profiles settings.register_profile("fast", max_examples=5, deadline=5000) settings.register_profile("slow", max_examples=20, deadline=5000) # Ignore the following modules because wsgi module fails as no # configuration file is found (--doctest-modules forces the module # loading) collect_ignore = ["swh/indexer/storage/api/wsgi.py"] # we use the swh_scheduler fixture pytest_plugins = ["swh.scheduler.pytest_plugin"] + + +@pytest.fixture(scope="session") +def swh_scheduler_celery_includes(swh_scheduler_celery_includes): + return swh_scheduler_celery_includes + [ + "swh.indexer.tasks", + ] diff --git a/requirements-swh.txt b/requirements-swh.txt index 3fb22d5..39e073c 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,6 +1,6 @@ swh.core[db,http] >= 0.2.2 swh.model >= 0.0.15 swh.objstorage >= 0.0.43 -swh.scheduler >= 0.0.47 +swh.scheduler >= 0.5.2 swh.storage >= 0.12.0 swh.journal >= 0.1.0 diff --git a/requirements-test.txt b/requirements-test.txt index ac0c1f0..c1f90dc 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,6 @@ confluent-kafka pytest +pytest-mock hypothesis>=3.11.0 swh.scheduler[testing] >= 0.5.0 swh.storage[testing] >= 0.10.0 diff --git a/swh/indexer/tasks.py b/swh/indexer/tasks.py index 3415fb1..084a896 100644 --- a/swh/indexer/tasks.py +++ b/swh/indexer/tasks.py @@ -1,48 +1,48 @@ # Copyright (C) 2016-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from celery import current_app as app +from celery import shared_task from .mimetype import MimetypeIndexer, MimetypePartitionIndexer from .ctags import CtagsIndexer from .fossology_license import FossologyLicenseIndexer, FossologyLicensePartitionIndexer from .rehash import RecomputeChecksums from .metadata import OriginMetadataIndexer -@app.task(name=__name__ + ".OriginMetadata") +@shared_task(name=__name__ + ".OriginMetadata") def origin_metadata(*args, **kwargs): return OriginMetadataIndexer().run(*args, **kwargs) -@app.task(name=__name__ + ".Ctags") +@shared_task(name=__name__ + ".Ctags") def ctags(*args, **kwargs): return CtagsIndexer().run(*args, **kwargs) -@app.task(name=__name__ + ".ContentFossologyLicense") +@shared_task(name=__name__ + ".ContentFossologyLicense") def fossology_license(*args, **kwargs): return FossologyLicenseIndexer().run(*args, **kwargs) -@app.task(name=__name__ + ".RecomputeChecksums") +@shared_task(name=__name__ + ".RecomputeChecksums") def recompute_checksums(*args, **kwargs): return RecomputeChecksums().run(*args, **kwargs) -@app.task(name=__name__ + ".ContentMimetype") +@shared_task(name=__name__ + ".ContentMimetype") def mimetype(*args, **kwargs): return MimetypeIndexer().run(*args, **kwargs) -@app.task(name=__name__ + ".ContentRangeMimetype") +@shared_task(name=__name__ + ".ContentRangeMimetype") def range_mimetype(*args, **kwargs): return MimetypePartitionIndexer().run(*args, **kwargs) -@app.task(name=__name__ + ".ContentRangeFossologyLicense") +@shared_task(name=__name__ + ".ContentRangeFossologyLicense") def range_license(*args, **kwargs): return FossologyLicensePartitionIndexer().run(*args, **kwargs) diff --git a/swh/indexer/tests/conftest.py b/swh/indexer/tests/conftest.py index 1ba1528..6438b89 100644 --- a/swh/indexer/tests/conftest.py +++ b/swh/indexer/tests/conftest.py @@ -1,74 +1,101 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import os + from datetime import timedelta from unittest.mock import patch +import yaml import pytest from swh.objstorage import get_objstorage from swh.storage import get_storage from swh.indexer.storage import get_indexer_storage from .utils import fill_storage, fill_obj_storage TASK_NAMES = ["revision_intrinsic_metadata", "origin_intrinsic_metadata"] @pytest.fixture def indexer_scheduler(swh_scheduler): for taskname in TASK_NAMES: swh_scheduler.create_task_type( { "type": taskname, "description": "The {} indexer testing task".format(taskname), "backend_name": "swh.indexer.tests.tasks.{}".format(taskname), "default_interval": timedelta(days=1), "min_interval": timedelta(hours=6), "max_interval": timedelta(days=12), "num_retries": 3, } ) return swh_scheduler @pytest.fixture def idx_storage(): """An instance of in-memory indexer storage that gets injected into all indexers classes. """ idx_storage = get_indexer_storage("memory", {}) with patch("swh.indexer.storage.in_memory.IndexerStorage") as idx_storage_mock: idx_storage_mock.return_value = idx_storage yield idx_storage @pytest.fixture def storage(): """An instance of in-memory storage that gets injected into all indexers classes. """ storage = get_storage(cls="memory") fill_storage(storage) with patch("swh.storage.in_memory.InMemoryStorage") as storage_mock: storage_mock.return_value = storage yield storage @pytest.fixture def obj_storage(): """An instance of in-memory objstorage that gets injected into all indexers classes. """ objstorage = get_objstorage("memory", {}) fill_obj_storage(objstorage) with patch.dict( "swh.objstorage.factory._STORAGE_CLASSES", {"memory": lambda: objstorage} ): yield objstorage + + +@pytest.fixture +def swh_indexer_config(): + return { + "storage": {"cls": "memory"}, + "objstorage": {"cls": "memory", "args": {},}, + "indexer_storage": {"cls": "memory", "args": {},}, + "tools": { + "name": "file", + "version": "1:5.30-1+deb9u1", + "configuration": {"type": "library", "debian-package": "python3-magic"}, + }, + "compute_checksums": ["blake2b512"], # for rehash indexer + } + + +@pytest.fixture +def swh_config(swh_indexer_config, monkeypatch, tmp_path): + conffile = os.path.join(str(tmp_path), "indexer.yml") + with open(conffile, "w") as f: + f.write(yaml.dump(swh_indexer_config)) + monkeypatch.setenv("SWH_CONFIG_FILENAME", conffile) + return conffile diff --git a/swh/indexer/tests/test_tasks.py b/swh/indexer/tests/test_tasks.py new file mode 100644 index 0000000..084581e --- /dev/null +++ b/swh/indexer/tests/test_tasks.py @@ -0,0 +1,123 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +def test_task_origin_metadata( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch("swh.indexer.tasks.OriginMetadataIndexer.run") + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.indexer.tasks.OriginMetadata", args=["origin-url"], + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + + +def test_task_ctags( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch("swh.indexer.tasks.CtagsIndexer.run") + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task("swh.indexer.tasks.Ctags", args=["id0"],) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + + +def test_task_fossology_license( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch("swh.indexer.tasks.FossologyLicenseIndexer.run") + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.indexer.tasks.ContentFossologyLicense", args=["id0"], + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + + +def test_task_recompute_checksums( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch("swh.indexer.tasks.RecomputeChecksums.run") + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.indexer.tasks.RecomputeChecksums", args=[[{"blake2b256": "id"}]], + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + + +def test_task_mimetype( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch("swh.indexer.tasks.MimetypeIndexer.run") + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.indexer.tasks.ContentMimetype", args=["id0"], + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + + +def test_task_range_mimetype( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch("swh.indexer.tasks.MimetypePartitionIndexer.run") + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.indexer.tasks.ContentRangeMimetype", args=[0, 4], + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + + +def test_task_range_license( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch( + "swh.indexer.tasks.FossologyLicensePartitionIndexer.run" + ) + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.indexer.tasks.ContentRangeFossologyLicense", args=[0, 4], + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"}