diff --git a/swh/indexer/tasks.py b/swh/indexer/tasks.py --- a/swh/indexer/tasks.py +++ b/swh/indexer/tasks.py @@ -3,9 +3,8 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import logging -from swh.scheduler.task import Task as SchedulerTask +from celery import current_app as app from .mimetype import MimetypeIndexer, MimetypeRangeIndexer from .language import LanguageIndexer @@ -17,103 +16,62 @@ from .metadata import RevisionMetadataIndexer, OriginMetadataIndexer from .origin_head import OriginHeadIndexer -logging.basicConfig(level=logging.INFO) +@app.task(name=__name__ + '.RevisionMetadata') +def revision_metadata(*args, **kwargs): + results = RevisionMetadataIndexer().run(*args, **kwargs) + return getattr(results, 'results', results) -class Task(SchedulerTask): - """Task whose results is needed for other computations. - """ - def run_task(self, *args, **kwargs): - indexer = self.Indexer().run(*args, **kwargs) - if hasattr(indexer, 'results'): # indexer tasks - return indexer.results - return indexer +@app.task(name=__name__ + '.OriginMetadata') +def origin_metadata(*args, **kwargs): + results = OriginMetadataIndexer().run(*args, **kwargs) + return getattr(results, 'results', results) -class StatusTask(SchedulerTask): - """Task which returns a status either eventful or uneventful. +@app.task(name=__name__ + '.OriginHead') +def origin_head(*args, **kwargs): + results = OriginHeadIndexer().run(*args, **kwargs) + return getattr(results, 'results', results) - """ - def run_task(self, *args, **kwargs): - results = self.Indexer().run(*args, **kwargs) - return {'status': 'eventful' if results else 'uneventful'} +@app.task(name=__name__ + '.ContentLanguage') +def content_language(*args, **kwargs): + results = LanguageIndexer().run(*args, **kwargs) + return getattr(results, 'results', results) -class RevisionMetadata(Task): - task_queue = 'swh_indexer_revision_metadata' - serializer = 'msgpack' +@app.task(name=__name__ + '.Ctags') +def ctags(*args, **kwargs): + results = CtagsIndexer().run(*args, **kwargs) + return getattr(results, 'results', results) - Indexer = RevisionMetadataIndexer +@app.task(name=__name__ + '.ContentFossologyLicense') +def fossology_license(*args, **kwargs): + results = FossologyLicenseIndexer().run(*args, **kwargs) + return getattr(results, 'results', results) -class OriginMetadata(Task): - task_queue = 'swh_indexer_origin_intrinsic_metadata' - Indexer = OriginMetadataIndexer +@app.task(name=__name__ + '.RecomputeChecksums') +def recompute_checksums(*args, **kwargs): + results = RecomputeChecksums().run(*args, **kwargs) + return getattr(results, 'results', results) -class OriginHead(Task): - task_queue = 'swh_indexer_origin_head' +@app.task(name=__name__ + '.ContentMimetype') +def mimetype(*args, **kwargs): + results = MimetypeIndexer().run(*args, **kwargs) + return {'status': 'eventful' if results else 'uneventful'} - Indexer = OriginHeadIndexer +@app.task(name=__name__ + '.ContentRangeMimetype') +def range_mimetype(*args, **kwargs): + results = MimetypeRangeIndexer(*args, **kwargs) + return {'status': 'eventful' if results else 'uneventful'} -class ContentMimetype(StatusTask): - """Compute (mimetype, encoding) on a list of sha1s' content. - """ - task_queue = 'swh_indexer_content_mimetype' - Indexer = MimetypeIndexer - - -class ContentRangeMimetype(StatusTask): - """Compute (mimetype, encoding) on a range of sha1s. - - """ - task_queue = 'swh_indexer_content_mimetype_range' - Indexer = MimetypeRangeIndexer - - -class ContentLanguage(Task): - """Task which computes the language from the sha1's content. - - """ - task_queue = 'swh_indexer_content_language' - - Indexer = LanguageIndexer - - -class Ctags(Task): - """Task which computes ctags from the sha1's content. - - """ - task_queue = 'swh_indexer_content_ctags' - - Indexer = CtagsIndexer - - -class ContentFossologyLicense(Task): - """Compute fossology licenses on a list of sha1s' content. - - """ - task_queue = 'swh_indexer_content_fossology_license' - Indexer = FossologyLicenseIndexer - - -class ContentRangeFossologyLicense(StatusTask): - """Compute fossology license on a range of sha1s. - - """ - task_queue = 'swh_indexer_content_fossology_license_range' - Indexer = FossologyLicenseRangeIndexer - - -class RecomputeChecksums(Task): - """Task which recomputes hashes and possibly new ones. - - """ - task_queue = 'swh_indexer_content_rehash' - - Indexer = RecomputeChecksums +@app.task(name=__name__ + '.ContentRangeFossologyLicense') +def range_license(*args, **kwargs): + results = FossologyLicenseRangeIndexer(*args, **kwargs) + return {'status': 'eventful' if results else 'uneventful'} diff --git a/swh/indexer/tests/conftest.py b/swh/indexer/tests/conftest.py --- a/swh/indexer/tests/conftest.py +++ b/swh/indexer/tests/conftest.py @@ -25,4 +25,5 @@ def celery_includes(): return [ 'swh.indexer.tests.tasks', + 'swh.indexer.tasks', ] diff --git a/swh/indexer/tests/tasks.py b/swh/indexer/tests/tasks.py --- a/swh/indexer/tests/tasks.py +++ b/swh/indexer/tests/tasks.py @@ -1,9 +1,10 @@ -from swh.scheduler.celery_backend.config import app +from celery import current_app as app + from swh.indexer.metadata import ( OriginMetadataIndexer, RevisionMetadataIndexer ) from .test_metadata import ContentMetadataTestIndexer -from .test_utils import BASE_TEST_CONFIG +from .utils import BASE_TEST_CONFIG class RevisionMetadataTestIndexer(RevisionMetadataIndexer): diff --git a/swh/indexer/tests/test_ctags.py b/swh/indexer/tests/test_ctags.py --- a/swh/indexer/tests/test_ctags.py +++ b/swh/indexer/tests/test_ctags.py @@ -12,7 +12,7 @@ CtagsIndexer, run_ctags ) -from swh.indexer.tests.test_utils import ( +from swh.indexer.tests.utils import ( CommonContentIndexerTest, CommonIndexerWithErrorsTest, CommonIndexerNoTool, SHA1_TO_CTAGS, NoDiskIndexer, BASE_TEST_CONFIG, diff --git a/swh/indexer/tests/test_fossology_license.py b/swh/indexer/tests/test_fossology_license.py --- a/swh/indexer/tests/test_fossology_license.py +++ b/swh/indexer/tests/test_fossology_license.py @@ -12,7 +12,7 @@ compute_license ) -from swh.indexer.tests.test_utils import ( +from swh.indexer.tests.utils import ( SHA1_TO_LICENSES, CommonContentIndexerTest, CommonContentIndexerRangeTest, CommonIndexerWithErrorsTest, CommonIndexerNoTool, NoDiskIndexer, BASE_TEST_CONFIG, fill_storage, fill_obj_storage diff --git a/swh/indexer/tests/test_language.py b/swh/indexer/tests/test_language.py --- a/swh/indexer/tests/test_language.py +++ b/swh/indexer/tests/test_language.py @@ -6,7 +6,7 @@ import unittest from swh.indexer import language from swh.indexer.language import LanguageIndexer -from swh.indexer.tests.test_utils import ( +from swh.indexer.tests.utils import ( CommonContentIndexerTest, CommonIndexerWithErrorsTest, CommonIndexerNoTool, BASE_TEST_CONFIG, fill_storage, fill_obj_storage ) diff --git a/swh/indexer/tests/test_metadata.py b/swh/indexer/tests/test_metadata.py --- a/swh/indexer/tests/test_metadata.py +++ b/swh/indexer/tests/test_metadata.py @@ -15,7 +15,7 @@ ContentMetadataIndexer, RevisionMetadataIndexer ) -from .test_utils import ( +from .utils import ( BASE_TEST_CONFIG, fill_obj_storage, fill_storage ) @@ -639,13 +639,13 @@ Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Description: swh-core ======== - + core library for swh's modules: - config parser - hash computations - serialization - logging mechanism - + Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers diff --git a/swh/indexer/tests/test_mimetype.py b/swh/indexer/tests/test_mimetype.py --- a/swh/indexer/tests/test_mimetype.py +++ b/swh/indexer/tests/test_mimetype.py @@ -9,7 +9,7 @@ MimetypeIndexer, MimetypeRangeIndexer, compute_mimetype_encoding ) -from swh.indexer.tests.test_utils import ( +from swh.indexer.tests.utils import ( CommonContentIndexerTest, CommonContentIndexerRangeTest, CommonIndexerWithErrorsTest, CommonIndexerNoTool, BASE_TEST_CONFIG, fill_storage, fill_obj_storage diff --git a/swh/indexer/tests/test_origin_head.py b/swh/indexer/tests/test_origin_head.py --- a/swh/indexer/tests/test_origin_head.py +++ b/swh/indexer/tests/test_origin_head.py @@ -6,7 +6,7 @@ import unittest from swh.indexer.origin_head import OriginHeadIndexer -from swh.indexer.tests.test_utils import ( +from swh.indexer.tests.utils import ( BASE_TEST_CONFIG, fill_storage ) diff --git a/swh/indexer/tests/test_origin_metadata.py b/swh/indexer/tests/test_origin_metadata.py --- a/swh/indexer/tests/test_origin_metadata.py +++ b/swh/indexer/tests/test_origin_metadata.py @@ -15,7 +15,7 @@ from swh.scheduler.celery_backend.runner import run_ready_tasks -from .test_utils import fill_storage, fill_obj_storage +from .utils import fill_storage, fill_obj_storage from .test_origin_head import OriginHeadTestIndexer from swh.indexer.tests.tasks import ( RevisionMetadataTestIndexer, OriginMetadataTestIndexer) diff --git a/swh/indexer/tests/test_utils.py b/swh/indexer/tests/utils.py rename from swh/indexer/tests/test_utils.py rename to swh/indexer/tests/utils.py