diff --git a/PKG-INFO b/PKG-INFO index 9ebd27a..20eaf96 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,69 +1,69 @@ Metadata-Version: 2.1 Name: swh.indexer -Version: 0.0.138 +Version: 0.0.139 Summary: Software Heritage Content Indexer Home-page: https://forge.softwareheritage.org/diffusion/78/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN -Project-URL: Source, https://forge.softwareheritage.org/source/swh-indexer Project-URL: Funding, https://www.softwareheritage.org/donate +Project-URL: Source, https://forge.softwareheritage.org/source/swh-indexer Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Description: swh-indexer ============ Tools to compute multiple indexes on SWH's raw contents: - content: - mimetype - ctags - language - fossology-license - metadata - revision: - metadata An indexer is in charge of: - looking up objects - extracting information from those objects - store those information in the swh-indexer db There are multiple indexers working on different object types: - content indexer: works with content sha1 hashes - revision indexer: works with revision sha1 hashes - origin indexer: works with origin identifiers Indexation procedure: - receive batch of ids - retrieve the associated data depending on object type - compute for that object some index - store the result to swh's storage Current content indexers: - mimetype (queue swh_indexer_content_mimetype): detect the encoding and mimetype - language (queue swh_indexer_content_language): detect the programming language - ctags (queue swh_indexer_content_ctags): compute tags information - fossology-license (queue swh_indexer_fossology_license): compute the license - metadata: translate file into translated_metadata dict Current revision indexers: - metadata: detects files containing metadata and retrieves translated_metadata in content_metadata table in storage or run content indexer to translate files. Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown Provides-Extra: testing diff --git a/swh.indexer.egg-info/PKG-INFO b/swh.indexer.egg-info/PKG-INFO index 9ebd27a..20eaf96 100644 --- a/swh.indexer.egg-info/PKG-INFO +++ b/swh.indexer.egg-info/PKG-INFO @@ -1,69 +1,69 @@ Metadata-Version: 2.1 Name: swh.indexer -Version: 0.0.138 +Version: 0.0.139 Summary: Software Heritage Content Indexer Home-page: https://forge.softwareheritage.org/diffusion/78/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN -Project-URL: Source, https://forge.softwareheritage.org/source/swh-indexer Project-URL: Funding, https://www.softwareheritage.org/donate +Project-URL: Source, https://forge.softwareheritage.org/source/swh-indexer Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Description: swh-indexer ============ Tools to compute multiple indexes on SWH's raw contents: - content: - mimetype - ctags - language - fossology-license - metadata - revision: - metadata An indexer is in charge of: - looking up objects - extracting information from those objects - store those information in the swh-indexer db There are multiple indexers working on different object types: - content indexer: works with content sha1 hashes - revision indexer: works with revision sha1 hashes - origin indexer: works with origin identifiers Indexation procedure: - receive batch of ids - retrieve the associated data depending on object type - compute for that object some index - store the result to swh's storage Current content indexers: - mimetype (queue swh_indexer_content_mimetype): detect the encoding and mimetype - language (queue swh_indexer_content_language): detect the programming language - ctags (queue swh_indexer_content_ctags): compute tags information - fossology-license (queue swh_indexer_fossology_license): compute the license - metadata: translate file into translated_metadata dict Current revision indexers: - metadata: detects files containing metadata and retrieves translated_metadata in content_metadata table in storage or run content indexer to translate files. Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown Provides-Extra: testing diff --git a/swh/indexer/tasks.py b/swh/indexer/tasks.py index f123500..97f921c 100644 --- a/swh/indexer/tasks.py +++ b/swh/indexer/tasks.py @@ -1,79 +1,64 @@ # Copyright (C) 2016-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from celery import current_app as app from .mimetype import MimetypeIndexer, MimetypeRangeIndexer from .language import LanguageIndexer from .ctags import CtagsIndexer from .fossology_license import ( FossologyLicenseIndexer, FossologyLicenseRangeIndexer ) from .rehash import RecomputeChecksums -from .metadata import ( - RevisionMetadataIndexer, OriginMetadataIndexer -) -from .origin_head import OriginHeadIndexer - - -@app.task(name=__name__ + '.RevisionMetadata') -def revision_metadata(*args, **kwargs): - results = RevisionMetadataIndexer().run(*args, **kwargs) - return getattr(results, 'results', results) +from .metadata import OriginMetadataIndexer @app.task(name=__name__ + '.OriginMetadata') def origin_metadata(*args, **kwargs): results = OriginMetadataIndexer().run(*args, **kwargs) return getattr(results, 'results', results) -@app.task(name=__name__ + '.OriginHead') -def origin_head(*args, **kwargs): - results = OriginHeadIndexer().run(*args, **kwargs) - return getattr(results, 'results', results) - - @app.task(name=__name__ + '.ContentLanguage') def content_language(*args, **kwargs): results = LanguageIndexer().run(*args, **kwargs) return getattr(results, 'results', results) @app.task(name=__name__ + '.Ctags') def ctags(*args, **kwargs): results = CtagsIndexer().run(*args, **kwargs) return getattr(results, 'results', results) @app.task(name=__name__ + '.ContentFossologyLicense') def fossology_license(*args, **kwargs): results = FossologyLicenseIndexer().run(*args, **kwargs) return getattr(results, 'results', results) @app.task(name=__name__ + '.RecomputeChecksums') def recompute_checksums(*args, **kwargs): results = RecomputeChecksums().run(*args, **kwargs) return getattr(results, 'results', results) @app.task(name=__name__ + '.ContentMimetype') def mimetype(*args, **kwargs): results = MimetypeIndexer().run(*args, **kwargs) return {'status': 'eventful' if results else 'uneventful'} @app.task(name=__name__ + '.ContentRangeMimetype') def range_mimetype(*args, **kwargs): results = MimetypeRangeIndexer().run(*args, **kwargs) return {'status': 'eventful' if results else 'uneventful'} @app.task(name=__name__ + '.ContentRangeFossologyLicense') def range_license(*args, **kwargs): results = FossologyLicenseRangeIndexer().run(*args, **kwargs) return {'status': 'eventful' if results else 'uneventful'} diff --git a/version.txt b/version.txt index 1768319..791e0c6 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.138-0-gd3684b8 \ No newline at end of file +v0.0.139-0-g8e12a00 \ No newline at end of file