diff --git a/conftest.py b/conftest.py --- a/conftest.py +++ b/conftest.py @@ -3,6 +3,8 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import pytest + from hypothesis import settings # define tests profile. Full documentation is at: @@ -17,3 +19,10 @@ # we use the swh_scheduler fixture pytest_plugins = ["swh.scheduler.pytest_plugin"] + + +@pytest.fixture(scope="session") +def swh_scheduler_celery_includes(swh_scheduler_celery_includes): + return swh_scheduler_celery_includes + [ + "swh.indexer.tasks", + ] diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,6 +1,6 @@ swh.core[db,http] >= 0.2.2 swh.model >= 0.0.15 swh.objstorage >= 0.0.43 -swh.scheduler >= 0.0.47 +swh.scheduler >= 0.5.2 swh.storage >= 0.12.0 swh.journal >= 0.1.0 diff --git a/requirements-test.txt b/requirements-test.txt --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,6 @@ confluent-kafka pytest +pytest-mock hypothesis>=3.11.0 swh.scheduler[testing] >= 0.5.0 swh.storage[testing] >= 0.10.0 diff --git a/swh/indexer/tasks.py b/swh/indexer/tasks.py --- a/swh/indexer/tasks.py +++ b/swh/indexer/tasks.py @@ -4,7 +4,7 @@ # See top-level LICENSE file for more information -from celery import current_app as app +from celery import shared_task from .mimetype import MimetypeIndexer, MimetypePartitionIndexer from .ctags import CtagsIndexer @@ -13,36 +13,36 @@ from .metadata import OriginMetadataIndexer -@app.task(name=__name__ + ".OriginMetadata") +@shared_task(name=__name__ + ".OriginMetadata") def origin_metadata(*args, **kwargs): return OriginMetadataIndexer().run(*args, **kwargs) -@app.task(name=__name__ + ".Ctags") +@shared_task(name=__name__ + ".Ctags") def ctags(*args, **kwargs): return CtagsIndexer().run(*args, **kwargs) -@app.task(name=__name__ + ".ContentFossologyLicense") +@shared_task(name=__name__ + ".ContentFossologyLicense") def fossology_license(*args, **kwargs): return FossologyLicenseIndexer().run(*args, **kwargs) -@app.task(name=__name__ + ".RecomputeChecksums") +@shared_task(name=__name__ + ".RecomputeChecksums") def recompute_checksums(*args, **kwargs): return RecomputeChecksums().run(*args, **kwargs) -@app.task(name=__name__ + ".ContentMimetype") +@shared_task(name=__name__ + ".ContentMimetype") def mimetype(*args, **kwargs): return MimetypeIndexer().run(*args, **kwargs) -@app.task(name=__name__ + ".ContentRangeMimetype") +@shared_task(name=__name__ + ".ContentRangeMimetype") def range_mimetype(*args, **kwargs): return MimetypePartitionIndexer().run(*args, **kwargs) -@app.task(name=__name__ + ".ContentRangeFossologyLicense") +@shared_task(name=__name__ + ".ContentRangeFossologyLicense") def range_license(*args, **kwargs): return FossologyLicensePartitionIndexer().run(*args, **kwargs) diff --git a/swh/indexer/tests/conftest.py b/swh/indexer/tests/conftest.py --- a/swh/indexer/tests/conftest.py +++ b/swh/indexer/tests/conftest.py @@ -3,9 +3,12 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import os + from datetime import timedelta from unittest.mock import patch +import yaml import pytest from swh.objstorage import get_objstorage @@ -72,3 +75,27 @@ "swh.objstorage.factory._STORAGE_CLASSES", {"memory": lambda: objstorage} ): yield objstorage + + +@pytest.fixture +def swh_indexer_config(): + return { + "storage": {"cls": "memory"}, + "objstorage": {"cls": "memory", "args": {},}, + "indexer_storage": {"cls": "memory", "args": {},}, + "tools": { + "name": "file", + "version": "1:5.30-1+deb9u1", + "configuration": {"type": "library", "debian-package": "python3-magic"}, + }, + "compute_checksums": ["blake2b512"], # for rehash indexer + } + + +@pytest.fixture +def swh_config(swh_indexer_config, monkeypatch, tmp_path): + conffile = os.path.join(str(tmp_path), "loader.yml") + with open(conffile, "w") as f: + f.write(yaml.dump(swh_indexer_config)) + monkeypatch.setenv("SWH_CONFIG_FILENAME", conffile) + return conffile diff --git a/swh/indexer/tests/test_tasks.py b/swh/indexer/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/indexer/tests/test_tasks.py @@ -0,0 +1,123 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +def test_task_origin_metadata( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch("swh.indexer.tasks.OriginMetadataIndexer.run") + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.indexer.tasks.OriginMetadata", args=["origin-url"], + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + + +def test_task_ctags( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch("swh.indexer.tasks.CtagsIndexer.run") + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task("swh.indexer.tasks.Ctags", args=["id0"],) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + + +def test_task_fossology_license( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch("swh.indexer.tasks.FossologyLicenseIndexer.run") + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.indexer.tasks.ContentFossologyLicense", args=["id0"], + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + + +def test_task_recompute_checksums( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch("swh.indexer.tasks.RecomputeChecksums.run") + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.indexer.tasks.RecomputeChecksums", args=[[{"blake2b256": "id"}]], + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + + +def test_task_mimetype( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch("swh.indexer.tasks.MimetypeIndexer.run") + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.indexer.tasks.ContentMimetype", args=["id0"], + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + + +def test_task_range_mimetype( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch("swh.indexer.tasks.MimetypePartitionIndexer.run") + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.indexer.tasks.ContentRangeMimetype", args=[0, 4], + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + + +def test_task_range_license( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + + mock_indexer = mocker.patch( + "swh.indexer.tasks.FossologyLicensePartitionIndexer.run" + ) + mock_indexer.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.indexer.tasks.ContentRangeFossologyLicense", args=[0, 4], + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"}