diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,4 +1,4 @@ -swh.core >= 0.0.40 +swh.core >= 0.0.41 swh.model >= 0.0.15 swh.objstorage >= 0.0.13 swh.scheduler >= 0.0.14 diff --git a/swh/indexer/storage/__init__.py b/swh/indexer/storage/__init__.py --- a/swh/indexer/storage/__init__.py +++ b/swh/indexer/storage/__init__.py @@ -9,6 +9,7 @@ from collections import defaultdict +from swh.core.api import remote_api_endpoint from swh.storage.common import db_transaction_generator, db_transaction from swh.storage.exc import StorageDBError from .db import Db @@ -45,7 +46,7 @@ return IndexerStorage(**args) -class IndexerStorage(): +class IndexerStorage: """SWH Indexer Storage """ @@ -72,6 +73,7 @@ return self._db return Db.from_pool(self._pool) + @remote_api_endpoint('check_config') def check_config(self, *, check_write): """Check that the storage is configured and ready to go.""" # Check permissions on one of the tables @@ -89,6 +91,7 @@ return True + @remote_api_endpoint('content_mimetype/missing') @db_transaction_generator() def content_mimetype_missing(self, mimetypes, db=None, cur=None): """List mimetypes missing from storage. @@ -108,6 +111,7 @@ for obj in db.content_mimetype_missing_from_list(mimetypes, cur): yield obj[0] + @remote_api_endpoint('content_mimetype/add') @db_transaction() def content_mimetype_add(self, mimetypes, conflict_update=False, db=None, cur=None): @@ -132,6 +136,7 @@ cur) db.content_mimetype_add_from_temp(conflict_update, cur) + @remote_api_endpoint('content_mimetype') @db_transaction_generator() def content_mimetype_get(self, ids, db=None, cur=None): """Retrieve full content mimetype per ids. @@ -152,6 +157,7 @@ yield converters.db_to_mimetype( dict(zip(db.content_mimetype_cols, c))) + @remote_api_endpoint('content_language/missing') @db_transaction_generator() def content_language_missing(self, languages, db=None, cur=None): """List languages missing from storage. @@ -171,6 +177,7 @@ for obj in db.content_language_missing_from_list(languages, cur): yield obj[0] + @remote_api_endpoint('content_language') @db_transaction_generator() def content_language_get(self, ids, db=None, cur=None): """Retrieve full content language per ids. @@ -190,6 +197,7 @@ yield converters.db_to_language( dict(zip(db.content_language_cols, c))) + @remote_api_endpoint('content_language/add') @db_transaction() def content_language_add(self, languages, conflict_update=False, db=None, cur=None): @@ -219,6 +227,7 @@ db.content_language_add_from_temp(conflict_update, cur) + @remote_api_endpoint('content/ctags/missing') @db_transaction_generator() def content_ctags_missing(self, ctags, db=None, cur=None): """List ctags missing from storage. @@ -238,6 +247,7 @@ for obj in db.content_ctags_missing_from_list(ctags, cur): yield obj[0] + @remote_api_endpoint('content/ctags') @db_transaction_generator() def content_ctags_get(self, ids, db=None, cur=None): """Retrieve ctags per id. @@ -259,6 +269,7 @@ for c in db.content_ctags_get_from_list(ids, cur): yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c))) + @remote_api_endpoint('content/ctags/add') @db_transaction() def content_ctags_add(self, ctags, conflict_update=False, db=None, cur=None): @@ -288,6 +299,7 @@ db.content_ctags_add_from_temp(conflict_update, cur) + @remote_api_endpoint('content/ctags/search') @db_transaction_generator() def content_ctags_search(self, expression, limit=10, last_sha1=None, db=None, cur=None): @@ -306,6 +318,7 @@ cur=cur): yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj))) + @remote_api_endpoint('content/fossology_license') @db_transaction_generator() def content_fossology_license_get(self, ids, db=None, cur=None): """Retrieve licenses per id. @@ -331,6 +344,7 @@ for id_, facts in d.items(): yield {id_: facts} + @remote_api_endpoint('content/fossology_license/add') @db_transaction() def content_fossology_license_add(self, licenses, conflict_update=False, db=None, cur=None): @@ -364,6 +378,7 @@ cur=cur) db.content_fossology_license_add_from_temp(conflict_update, cur) + @remote_api_endpoint('content_metadata/missing') @db_transaction_generator() def content_metadata_missing(self, metadata, db=None, cur=None): """List metadata missing from storage. @@ -383,6 +398,7 @@ for obj in db.content_metadata_missing_from_list(metadata, cur): yield obj[0] + @remote_api_endpoint('content_metadata') @db_transaction_generator() def content_metadata_get(self, ids, db=None, cur=None): """Retrieve metadata per id. @@ -402,6 +418,7 @@ yield converters.db_to_metadata( dict(zip(db.content_metadata_cols, c))) + @remote_api_endpoint('content_metadata/add') @db_transaction() def content_metadata_add(self, metadata, conflict_update=False, db=None, cur=None): @@ -425,6 +442,7 @@ cur) db.content_metadata_add_from_temp(conflict_update, cur) + @remote_api_endpoint('revision_metadata/missing') @db_transaction_generator() def revision_metadata_missing(self, metadata, db=None, cur=None): """List metadata missing from storage. @@ -443,6 +461,7 @@ for obj in db.revision_metadata_missing_from_list(metadata, cur): yield obj[0] + @remote_api_endpoint('revision_metadata') @db_transaction_generator() def revision_metadata_get(self, ids, db=None, cur=None): """Retrieve revision metadata per id. @@ -462,6 +481,7 @@ yield converters.db_to_metadata( dict(zip(db.revision_metadata_cols, c))) + @remote_api_endpoint('revision_metadata/add') @db_transaction() def revision_metadata_add(self, metadata, conflict_update=False, db=None, cur=None): @@ -485,6 +505,7 @@ cur) db.revision_metadata_add_from_temp(conflict_update, cur) + @remote_api_endpoint('indexer_configuration/add') @db_transaction_generator() def indexer_configuration_add(self, tools, db=None, cur=None): """Add new tools to the storage. @@ -513,6 +534,7 @@ for line in tools: yield dict(zip(db.indexer_configuration_cols, line)) + @remote_api_endpoint('indexer_configuration/data') @db_transaction() def indexer_configuration_get(self, tool, db=None, cur=None): """Retrieve tool information. diff --git a/swh/indexer/storage/api/client.py b/swh/indexer/storage/api/client.py --- a/swh/indexer/storage/api/client.py +++ b/swh/indexer/storage/api/client.py @@ -3,99 +3,18 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information - from swh.core.api import SWHRemoteAPI from swh.storage.exc import StorageAPIError +from .. import IndexerStorage + class RemoteStorage(SWHRemoteAPI): """Proxy to a remote storage API""" + + backend_class = IndexerStorage + def __init__(self, url, timeout=None): super().__init__( api_exception=StorageAPIError, url=url, timeout=timeout) - - def check_config(self, *, check_write): - return self.post('check_config', {'check_write': check_write}) - - def content_mimetype_add(self, mimetypes, conflict_update=False): - return self.post('content_mimetype/add', { - 'mimetypes': mimetypes, - 'conflict_update': conflict_update, - }) - - def content_mimetype_missing(self, mimetypes): - return self.post('content_mimetype/missing', {'mimetypes': mimetypes}) - - def content_mimetype_get(self, ids): - return self.post('content_mimetype', {'ids': ids}) - - def content_language_add(self, languages, conflict_update=False): - return self.post('content_language/add', { - 'languages': languages, - 'conflict_update': conflict_update, - }) - - def content_language_missing(self, languages): - return self.post('content_language/missing', {'languages': languages}) - - def content_language_get(self, ids): - return self.post('content_language', {'ids': ids}) - - def content_ctags_add(self, ctags, conflict_update=False): - return self.post('content/ctags/add', { - 'ctags': ctags, - 'conflict_update': conflict_update, - }) - - def content_ctags_missing(self, ctags): - return self.post('content/ctags/missing', {'ctags': ctags}) - - def content_ctags_get(self, ids): - return self.post('content/ctags', {'ids': ids}) - - def content_ctags_search(self, expression, limit=10, last_sha1=None): - return self.post('content/ctags/search', { - 'expression': expression, - 'limit': limit, - 'last_sha1': last_sha1, - }) - - def content_fossology_license_add(self, licenses, conflict_update=False): - return self.post('content/fossology_license/add', { - 'licenses': licenses, - 'conflict_update': conflict_update, - }) - - def content_fossology_license_get(self, ids): - return self.post('content/fossology_license', {'ids': ids}) - - def content_metadata_add(self, metadata, conflict_update=False): - return self.post('content_metadata/add', { - 'metadata': metadata, - 'conflict_update': conflict_update, - }) - - def content_metadata_missing(self, metadata): - return self.post('content_metadata/missing', {'metadata': metadata}) - - def content_metadata_get(self, ids): - return self.post('content_metadata', {'ids': ids}) - - def revision_metadata_add(self, metadata, conflict_update=False): - return self.post('revision_metadata/add', { - 'metadata': metadata, - 'conflict_update': conflict_update, - }) - - def revision_metadata_missing(self, metadata): - return self.post('revision_metadata/missing', {'metadata': metadata}) - - def revision_metadata_get(self, ids): - return self.post('revision_metadata', {'ids': ids}) - - def indexer_configuration_add(self, tools): - return self.post('indexer_configuration/add', {'tools': tools}) - - def indexer_configuration_get(self, tool): - return self.post('indexer_configuration/data', {'tool': tool}) diff --git a/swh/indexer/storage/api/server.py b/swh/indexer/storage/api/server.py --- a/swh/indexer/storage/api/server.py +++ b/swh/indexer/storage/api/server.py @@ -6,14 +6,13 @@ import logging import click -from flask import request - from swh.core import config -from swh.core.api import (SWHServerAPIApp, decode_request, - error_handler, +from swh.core.api import (SWHServerAPIApp, error_handler, encode_data_server as encode_data) from swh.indexer.storage import get_indexer_storage, INDEXER_CFG_KEY +from .. import IndexerStorage + DEFAULT_CONFIG_PATH = 'storage/indexer' DEFAULT_CONFIG = { @@ -26,15 +25,6 @@ } -app = SWHServerAPIApp(__name__) -storage = None - - -@app.errorhandler(Exception) -def my_error_handler(exception): - return error_handler(exception, encode_data) - - def get_storage(): global storage if not storage: @@ -43,134 +33,20 @@ return storage -@app.route('/') -def index(): - return 'SWH Indexer Storage API server' - - -@app.route('/check_config', methods=['POST']) -def check_config(): - return encode_data(get_storage().check_config(**decode_request(request))) - - -@app.route('/content_mimetype/add', methods=['POST']) -def content_mimetype_add(): - return encode_data( - get_storage().content_mimetype_add(**decode_request(request))) - - -@app.route('/content_mimetype/missing', methods=['POST']) -def content_mimetype_missing(): - return encode_data( - get_storage().content_mimetype_missing(**decode_request(request))) - - -@app.route('/content_mimetype', methods=['POST']) -def content_mimetype_get(): - return encode_data( - get_storage().content_mimetype_get(**decode_request(request))) - - -@app.route('/content_language/add', methods=['POST']) -def content_language_add(): - return encode_data( - get_storage().content_language_add(**decode_request(request))) - - -@app.route('/content_language/missing', methods=['POST']) -def content_language_missing(): - return encode_data( - get_storage().content_language_missing(**decode_request(request))) - - -@app.route('/content_language', methods=['POST']) -def content_language_get(): - return encode_data( - get_storage().content_language_get(**decode_request(request))) - - -@app.route('/content/ctags/add', methods=['POST']) -def content_ctags_add(): - return encode_data( - get_storage().content_ctags_add(**decode_request(request))) - - -@app.route('/content/ctags/search', methods=['POST']) -def content_ctags_search(): - return encode_data( - get_storage().content_ctags_search(**decode_request(request))) - - -@app.route('/content/ctags/missing', methods=['POST']) -def content_ctags_missing(): - return encode_data( - get_storage().content_ctags_missing(**decode_request(request))) - - -@app.route('/content/ctags', methods=['POST']) -def content_ctags_get(): - return encode_data( - get_storage().content_ctags_get(**decode_request(request))) - - -@app.route('/content/fossology_license/add', methods=['POST']) -def content_fossology_license_add(): - return encode_data( - get_storage().content_fossology_license_add(**decode_request(request))) - - -@app.route('/content/fossology_license', methods=['POST']) -def content_fossology_license_get(): - return encode_data( - get_storage().content_fossology_license_get(**decode_request(request))) - - -@app.route('/indexer_configuration/data', methods=['POST']) -def indexer_configuration_get(): - return encode_data(get_storage().indexer_configuration_get( - **decode_request(request))) - - -@app.route('/indexer_configuration/add', methods=['POST']) -def indexer_configuration_add(): - return encode_data(get_storage().indexer_configuration_add( - **decode_request(request))) - - -@app.route('/content_metadata/add', methods=['POST']) -def content_metadata_add(): - return encode_data( - get_storage().content_metadata_add(**decode_request(request))) - - -@app.route('/content_metadata/missing', methods=['POST']) -def content_metadata_missing(): - return encode_data( - get_storage().content_metadata_missing(**decode_request(request))) - - -@app.route('/content_metadata', methods=['POST']) -def content_metadata_get(): - return encode_data( - get_storage().content_metadata_get(**decode_request(request))) - - -@app.route('/revision_metadata/add', methods=['POST']) -def revision_metadata_add(): - return encode_data( - get_storage().revision_metadata_add(**decode_request(request))) +app = SWHServerAPIApp(__name__, + backend_class=IndexerStorage, + backend_factory=get_storage) +storage = None -@app.route('/revision_metadata/missing', methods=['POST']) -def revision_metadata_missing(): - return encode_data( - get_storage().revision_metadata_missing(**decode_request(request))) +@app.errorhandler(Exception) +def my_error_handler(exception): + return error_handler(exception, encode_data) -@app.route('/revision_metadata', methods=['POST']) -def revision_metadata_get(): - return encode_data( - get_storage().revision_metadata_get(**decode_request(request))) +@app.route('/') +def index(): + return 'SWH Indexer Storage API server' def run_from_webserver(environ, start_response,