Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9125134
D980.id3099.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
9 KB
Subscribers
None
D980.id3099.diff
View Options
diff --git a/swh/indexer/tasks.py b/swh/indexer/tasks.py
--- a/swh/indexer/tasks.py
+++ b/swh/indexer/tasks.py
@@ -3,9 +3,8 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-import logging
-from swh.scheduler.task import Task as SchedulerTask
+from celery import current_app as app
from .mimetype import MimetypeIndexer, MimetypeRangeIndexer
from .language import LanguageIndexer
@@ -17,103 +16,62 @@
from .metadata import RevisionMetadataIndexer, OriginMetadataIndexer
from .origin_head import OriginHeadIndexer
-logging.basicConfig(level=logging.INFO)
+@app.task(name=__name__ + '.RevisionMetadata')
+def revision_metadata(*args, **kwargs):
+ results = RevisionMetadataIndexer().run(*args, **kwargs)
+ return getattr(results, 'results', results)
-class Task(SchedulerTask):
- """Task whose results is needed for other computations.
- """
- def run_task(self, *args, **kwargs):
- indexer = self.Indexer().run(*args, **kwargs)
- if hasattr(indexer, 'results'): # indexer tasks
- return indexer.results
- return indexer
+@app.task(name=__name__ + '.OriginMetadata')
+def origin_metadata(*args, **kwargs):
+ results = OriginMetadataIndexer().run(*args, **kwargs)
+ return getattr(results, 'results', results)
-class StatusTask(SchedulerTask):
- """Task which returns a status either eventful or uneventful.
+@app.task(name=__name__ + '.OriginHead')
+def origin_head(*args, **kwargs):
+ results = OriginHeadIndexer().run(*args, **kwargs)
+ return getattr(results, 'results', results)
- """
- def run_task(self, *args, **kwargs):
- results = self.Indexer().run(*args, **kwargs)
- return {'status': 'eventful' if results else 'uneventful'}
+@app.task(name=__name__ + '.ContentLanguage')
+def content_language(*args, **kwargs):
+ results = LanguageIndexer().run(*args, **kwargs)
+ return getattr(results, 'results', results)
-class RevisionMetadata(Task):
- task_queue = 'swh_indexer_revision_metadata'
- serializer = 'msgpack'
+@app.task(name=__name__ + '.Ctags')
+def ctags(*args, **kwargs):
+ results = CtagsIndexer().run(*args, **kwargs)
+ return getattr(results, 'results', results)
- Indexer = RevisionMetadataIndexer
+@app.task(name=__name__ + '.ContentFossologyLicense')
+def fossology_license(*args, **kwargs):
+ results = FossologyLicenseIndexer().run(*args, **kwargs)
+ return getattr(results, 'results', results)
-class OriginMetadata(Task):
- task_queue = 'swh_indexer_origin_intrinsic_metadata'
- Indexer = OriginMetadataIndexer
+@app.task(name=__name__ + '.RecomputeChecksums')
+def recompute_checksums(*args, **kwargs):
+ results = RecomputeChecksums().run(*args, **kwargs)
+ return getattr(results, 'results', results)
-class OriginHead(Task):
- task_queue = 'swh_indexer_origin_head'
+@app.task(name=__name__ + '.ContentMimetype')
+def mimetype(*args, **kwargs):
+ results = MimetypeIndexer().run(*args, **kwargs)
+ return {'status': 'eventful' if results else 'uneventful'}
- Indexer = OriginHeadIndexer
+@app.task(name=__name__ + '.ContentRangeMimetype')
+def range_mimetype(*args, **kwargs):
+ results = MimetypeRangeIndexer(*args, **kwargs)
+ return {'status': 'eventful' if results else 'uneventful'}
-class ContentMimetype(StatusTask):
- """Compute (mimetype, encoding) on a list of sha1s' content.
- """
- task_queue = 'swh_indexer_content_mimetype'
- Indexer = MimetypeIndexer
-
-
-class ContentRangeMimetype(StatusTask):
- """Compute (mimetype, encoding) on a range of sha1s.
-
- """
- task_queue = 'swh_indexer_content_mimetype_range'
- Indexer = MimetypeRangeIndexer
-
-
-class ContentLanguage(Task):
- """Task which computes the language from the sha1's content.
-
- """
- task_queue = 'swh_indexer_content_language'
-
- Indexer = LanguageIndexer
-
-
-class Ctags(Task):
- """Task which computes ctags from the sha1's content.
-
- """
- task_queue = 'swh_indexer_content_ctags'
-
- Indexer = CtagsIndexer
-
-
-class ContentFossologyLicense(Task):
- """Compute fossology licenses on a list of sha1s' content.
-
- """
- task_queue = 'swh_indexer_content_fossology_license'
- Indexer = FossologyLicenseIndexer
-
-
-class ContentRangeFossologyLicense(StatusTask):
- """Compute fossology license on a range of sha1s.
-
- """
- task_queue = 'swh_indexer_content_fossology_license_range'
- Indexer = FossologyLicenseRangeIndexer
-
-
-class RecomputeChecksums(Task):
- """Task which recomputes hashes and possibly new ones.
-
- """
- task_queue = 'swh_indexer_content_rehash'
-
- Indexer = RecomputeChecksums
+@app.task(name=__name__ + '.ContentRangeFossologyLicense')
+def range_license(*args, **kwargs):
+ results = FossologyLicenseRangeIndexer(*args, **kwargs)
+ return {'status': 'eventful' if results else 'uneventful'}
diff --git a/swh/indexer/tests/conftest.py b/swh/indexer/tests/conftest.py
--- a/swh/indexer/tests/conftest.py
+++ b/swh/indexer/tests/conftest.py
@@ -25,4 +25,5 @@
def celery_includes():
return [
'swh.indexer.tests.tasks',
+ 'swh.indexer.tasks',
]
diff --git a/swh/indexer/tests/tasks.py b/swh/indexer/tests/tasks.py
--- a/swh/indexer/tests/tasks.py
+++ b/swh/indexer/tests/tasks.py
@@ -1,9 +1,10 @@
-from swh.scheduler.celery_backend.config import app
+from celery import current_app as app
+
from swh.indexer.metadata import (
OriginMetadataIndexer, RevisionMetadataIndexer
)
from .test_metadata import ContentMetadataTestIndexer
-from .test_utils import BASE_TEST_CONFIG
+from .utils import BASE_TEST_CONFIG
class RevisionMetadataTestIndexer(RevisionMetadataIndexer):
diff --git a/swh/indexer/tests/test_ctags.py b/swh/indexer/tests/test_ctags.py
--- a/swh/indexer/tests/test_ctags.py
+++ b/swh/indexer/tests/test_ctags.py
@@ -12,7 +12,7 @@
CtagsIndexer, run_ctags
)
-from swh.indexer.tests.test_utils import (
+from swh.indexer.tests.utils import (
CommonContentIndexerTest,
CommonIndexerWithErrorsTest, CommonIndexerNoTool,
SHA1_TO_CTAGS, NoDiskIndexer, BASE_TEST_CONFIG,
diff --git a/swh/indexer/tests/test_fossology_license.py b/swh/indexer/tests/test_fossology_license.py
--- a/swh/indexer/tests/test_fossology_license.py
+++ b/swh/indexer/tests/test_fossology_license.py
@@ -12,7 +12,7 @@
compute_license
)
-from swh.indexer.tests.test_utils import (
+from swh.indexer.tests.utils import (
SHA1_TO_LICENSES, CommonContentIndexerTest, CommonContentIndexerRangeTest,
CommonIndexerWithErrorsTest, CommonIndexerNoTool, NoDiskIndexer,
BASE_TEST_CONFIG, fill_storage, fill_obj_storage
diff --git a/swh/indexer/tests/test_language.py b/swh/indexer/tests/test_language.py
--- a/swh/indexer/tests/test_language.py
+++ b/swh/indexer/tests/test_language.py
@@ -6,7 +6,7 @@
import unittest
from swh.indexer import language
from swh.indexer.language import LanguageIndexer
-from swh.indexer.tests.test_utils import (
+from swh.indexer.tests.utils import (
CommonContentIndexerTest, CommonIndexerWithErrorsTest,
CommonIndexerNoTool, BASE_TEST_CONFIG, fill_storage, fill_obj_storage
)
diff --git a/swh/indexer/tests/test_metadata.py b/swh/indexer/tests/test_metadata.py
--- a/swh/indexer/tests/test_metadata.py
+++ b/swh/indexer/tests/test_metadata.py
@@ -15,7 +15,7 @@
ContentMetadataIndexer, RevisionMetadataIndexer
)
-from .test_utils import (
+from .utils import (
BASE_TEST_CONFIG, fill_obj_storage, fill_storage
)
diff --git a/swh/indexer/tests/test_mimetype.py b/swh/indexer/tests/test_mimetype.py
--- a/swh/indexer/tests/test_mimetype.py
+++ b/swh/indexer/tests/test_mimetype.py
@@ -9,7 +9,7 @@
MimetypeIndexer, MimetypeRangeIndexer, compute_mimetype_encoding
)
-from swh.indexer.tests.test_utils import (
+from swh.indexer.tests.utils import (
CommonContentIndexerTest, CommonContentIndexerRangeTest,
CommonIndexerWithErrorsTest, CommonIndexerNoTool,
BASE_TEST_CONFIG, fill_storage, fill_obj_storage
diff --git a/swh/indexer/tests/test_origin_head.py b/swh/indexer/tests/test_origin_head.py
--- a/swh/indexer/tests/test_origin_head.py
+++ b/swh/indexer/tests/test_origin_head.py
@@ -6,7 +6,7 @@
import unittest
from swh.indexer.origin_head import OriginHeadIndexer
-from swh.indexer.tests.test_utils import (
+from swh.indexer.tests.utils import (
BASE_TEST_CONFIG, fill_storage
)
diff --git a/swh/indexer/tests/test_origin_metadata.py b/swh/indexer/tests/test_origin_metadata.py
--- a/swh/indexer/tests/test_origin_metadata.py
+++ b/swh/indexer/tests/test_origin_metadata.py
@@ -15,7 +15,7 @@
from swh.scheduler.celery_backend.runner import run_ready_tasks
-from .test_utils import fill_storage, fill_obj_storage
+from .utils import fill_storage, fill_obj_storage
from .test_origin_head import OriginHeadTestIndexer
from swh.indexer.tests.tasks import (
RevisionMetadataTestIndexer, OriginMetadataTestIndexer)
diff --git a/swh/indexer/tests/test_utils.py b/swh/indexer/tests/utils.py
rename from swh/indexer/tests/test_utils.py
rename to swh/indexer/tests/utils.py
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Fri, Jun 20, 8:07 PM (3 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3227338
Attached To
D980: Rewrite celery tasks as decorated functions
Event Timeline
Log In to Comment