diff --git a/swh/indexer/storage/__init__.py b/swh/indexer/storage/__init__.py --- a/swh/indexer/storage/__init__.py +++ b/swh/indexer/storage/__init__.py @@ -19,6 +19,13 @@ INDEXER_CFG_KEY = 'indexer_storage' +def endpoint(path): + def dec(f): + f._endpoint_path = path + return f + return dec + + def get_indexer_storage(cls, args): """Get an indexer storage object of class `storage_class` with arguments `storage_args`. @@ -72,6 +79,7 @@ return self._db return Db.from_pool(self._pool) + @endpoint('check_config') def check_config(self, *, check_write): """Check that the storage is configured and ready to go.""" # Check permissions on one of the tables @@ -89,6 +97,7 @@ return True + @endpoint('content_mimetype/missing') @db_transaction_generator() def content_mimetype_missing(self, mimetypes, db=None, cur=None): """List mimetypes missing from storage. @@ -108,6 +117,7 @@ for obj in db.content_mimetype_missing_from_list(mimetypes, cur): yield obj[0] + @endpoint('content_mimetype/add') @db_transaction() def content_mimetype_add(self, mimetypes, conflict_update=False, db=None, cur=None): @@ -132,6 +142,7 @@ cur) db.content_mimetype_add_from_temp(conflict_update, cur) + @endpoint('content_mimetype') @db_transaction_generator() def content_mimetype_get(self, ids, db=None, cur=None): """Retrieve full content mimetype per ids. @@ -152,6 +163,7 @@ yield converters.db_to_mimetype( dict(zip(db.content_mimetype_cols, c))) + @endpoint('content_language/missing') @db_transaction_generator() def content_language_missing(self, languages, db=None, cur=None): """List languages missing from storage. @@ -171,6 +183,7 @@ for obj in db.content_language_missing_from_list(languages, cur): yield obj[0] + @endpoint('content_language') @db_transaction_generator() def content_language_get(self, ids, db=None, cur=None): """Retrieve full content language per ids. @@ -190,6 +203,7 @@ yield converters.db_to_language( dict(zip(db.content_language_cols, c))) + @endpoint('content_language/add') @db_transaction() def content_language_add(self, languages, conflict_update=False, db=None, cur=None): @@ -219,6 +233,7 @@ db.content_language_add_from_temp(conflict_update, cur) + @endpoint('content/ctags/missing') @db_transaction_generator() def content_ctags_missing(self, ctags, db=None, cur=None): """List ctags missing from storage. @@ -238,6 +253,7 @@ for obj in db.content_ctags_missing_from_list(ctags, cur): yield obj[0] + @endpoint('content/ctags') @db_transaction_generator() def content_ctags_get(self, ids, db=None, cur=None): """Retrieve ctags per id. @@ -259,6 +275,7 @@ for c in db.content_ctags_get_from_list(ids, cur): yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c))) + @endpoint('content/ctags/add') @db_transaction() def content_ctags_add(self, ctags, conflict_update=False, db=None, cur=None): @@ -288,6 +305,7 @@ db.content_ctags_add_from_temp(conflict_update, cur) + @endpoint('content/ctags/search') @db_transaction_generator() def content_ctags_search(self, expression, limit=10, last_sha1=None, db=None, cur=None): @@ -306,6 +324,7 @@ cur=cur): yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj))) + @endpoint('content/fossology_license') @db_transaction_generator() def content_fossology_license_get(self, ids, db=None, cur=None): """Retrieve licenses per id. @@ -331,6 +350,7 @@ for id_, facts in d.items(): yield {id_: facts} + @endpoint('content/fossology_license/add') @db_transaction() def content_fossology_license_add(self, licenses, conflict_update=False, db=None, cur=None): @@ -364,6 +384,7 @@ cur=cur) db.content_fossology_license_add_from_temp(conflict_update, cur) + @endpoint('content_metadata/missing') @db_transaction_generator() def content_metadata_missing(self, metadata, db=None, cur=None): """List metadata missing from storage. @@ -383,6 +404,7 @@ for obj in db.content_metadata_missing_from_list(metadata, cur): yield obj[0] + @endpoint('content_metadata') @db_transaction_generator() def content_metadata_get(self, ids, db=None, cur=None): """Retrieve metadata per id. @@ -402,6 +424,7 @@ yield converters.db_to_metadata( dict(zip(db.content_metadata_cols, c))) + @endpoint('content_metadata/add') @db_transaction() def content_metadata_add(self, metadata, conflict_update=False, db=None, cur=None): @@ -425,6 +448,7 @@ cur) db.content_metadata_add_from_temp(conflict_update, cur) + @endpoint('revision_metadata/missing') @db_transaction_generator() def revision_metadata_missing(self, metadata, db=None, cur=None): """List metadata missing from storage. @@ -443,6 +467,7 @@ for obj in db.revision_metadata_missing_from_list(metadata, cur): yield obj[0] + @endpoint('revision_metadata') @db_transaction_generator() def revision_metadata_get(self, ids, db=None, cur=None): """Retrieve revision metadata per id. @@ -462,6 +487,7 @@ yield converters.db_to_metadata( dict(zip(db.revision_metadata_cols, c))) + @endpoint('revision_metadata/add') @db_transaction() def revision_metadata_add(self, metadata, conflict_update=False, db=None, cur=None): @@ -485,6 +511,7 @@ cur) db.revision_metadata_add_from_temp(conflict_update, cur) + @endpoint('indexer_configuration/add') @db_transaction_generator() def indexer_configuration_add(self, tools, db=None, cur=None): """Add new tools to the storage. @@ -513,6 +540,7 @@ for line in tools: yield dict(zip(db.indexer_configuration_cols, line)) + @endpoint('indexer_configuration/data') @db_transaction() def indexer_configuration_get(self, tool, db=None, cur=None): """Retrieve tool information. diff --git a/swh/indexer/storage/api/client.py b/swh/indexer/storage/api/client.py --- a/swh/indexer/storage/api/client.py +++ b/swh/indexer/storage/api/client.py @@ -3,11 +3,26 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import functools +import inspect from swh.core.api import SWHRemoteAPI from swh.storage.exc import StorageAPIError +from .. import IndexerStorage + + +def post_to(endpoint_path): + def d(f): + @functools.wraps(f) + def newf(*args, **kwargs): + post_data = inspect.getcallargs(f, *args, **kwargs) + self = post_data.pop('self') + return self.post(endpoint_path, post_data) + return newf + return d + class RemoteStorage(SWHRemoteAPI): """Proxy to a remote storage API""" @@ -15,87 +30,30 @@ super().__init__( api_exception=StorageAPIError, url=url, timeout=timeout) - def check_config(self, *, check_write): - return self.post('check_config', {'check_write': check_write}) - - def content_mimetype_add(self, mimetypes, conflict_update=False): - return self.post('content_mimetype/add', { - 'mimetypes': mimetypes, - 'conflict_update': conflict_update, - }) - - def content_mimetype_missing(self, mimetypes): - return self.post('content_mimetype/missing', {'mimetypes': mimetypes}) - - def content_mimetype_get(self, ids): - return self.post('content_mimetype', {'ids': ids}) - - def content_language_add(self, languages, conflict_update=False): - return self.post('content_language/add', { - 'languages': languages, - 'conflict_update': conflict_update, - }) - - def content_language_missing(self, languages): - return self.post('content_language/missing', {'languages': languages}) - - def content_language_get(self, ids): - return self.post('content_language', {'ids': ids}) - - def content_ctags_add(self, ctags, conflict_update=False): - return self.post('content/ctags/add', { - 'ctags': ctags, - 'conflict_update': conflict_update, - }) - - def content_ctags_missing(self, ctags): - return self.post('content/ctags/missing', {'ctags': ctags}) - - def content_ctags_get(self, ids): - return self.post('content/ctags', {'ids': ids}) - - def content_ctags_search(self, expression, limit=10, last_sha1=None): - return self.post('content/ctags/search', { - 'expression': expression, - 'limit': limit, - 'last_sha1': last_sha1, - }) - - def content_fossology_license_add(self, licenses, conflict_update=False): - return self.post('content/fossology_license/add', { - 'licenses': licenses, - 'conflict_update': conflict_update, - }) - - def content_fossology_license_get(self, ids): - return self.post('content/fossology_license', {'ids': ids}) - - def content_metadata_add(self, metadata, conflict_update=False): - return self.post('content_metadata/add', { - 'metadata': metadata, - 'conflict_update': conflict_update, - }) - - def content_metadata_missing(self, metadata): - return self.post('content_metadata/missing', {'metadata': metadata}) - - def content_metadata_get(self, ids): - return self.post('content_metadata', {'ids': ids}) - - def revision_metadata_add(self, metadata, conflict_update=False): - return self.post('revision_metadata/add', { - 'metadata': metadata, - 'conflict_update': conflict_update, - }) - - def revision_metadata_missing(self, metadata): - return self.post('revision_metadata/missing', {'metadata': metadata}) - - def revision_metadata_get(self, ids): - return self.post('revision_metadata', {'ids': ids}) - - def indexer_configuration_add(self, tools): - return self.post('indexer_configuration/add', {'tools': tools}) - def indexer_configuration_get(self, tool): - return self.post('indexer_configuration/data', {'tool': tool}) +# For each method wrapped with @endpoint in IndexerStorage, add a new +# method in RemoteStorage, with the same documentation. +# +# Note that, despite the usage of decorator magic (eg. functools.wrap), +# this never actually calls an IndexerStorage method. +for (_name, _meth) in IndexerStorage.__dict__.items(): + if hasattr(_meth, '_endpoint_path'): + def _closure(name, meth): + wrapped_meth = inspect.unwrap(meth) + + @functools.wraps(meth) # Copy signature and doc + def _meth(*args, **kwargs): + # Match arguments and parameters + post_data = inspect.getcallargs(wrapped_meth, *args, **kwargs) + + # Remove arguments that should not be passed + self = post_data.pop('self') + post_data.pop('cur', None) + post_data.pop('db', None) + + # Send the request. + return self.post(meth._endpoint_path, post_data) + setattr(RemoteStorage, name, _meth) + _closure(_name, _meth) + +del _name, _meth, _closure diff --git a/swh/indexer/storage/api/server.py b/swh/indexer/storage/api/server.py --- a/swh/indexer/storage/api/server.py +++ b/swh/indexer/storage/api/server.py @@ -3,6 +3,7 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import functools import logging import click @@ -14,6 +15,8 @@ encode_data_server as encode_data) from swh.indexer.storage import get_indexer_storage, INDEXER_CFG_KEY +from .. import IndexerStorage + DEFAULT_CONFIG_PATH = 'storage/indexer' DEFAULT_CONFIG = { @@ -48,129 +51,20 @@ return 'SWH Indexer Storage API server' -@app.route('/check_config', methods=['POST']) -def check_config(): - return encode_data(get_storage().check_config(**decode_request(request))) - - -@app.route('/content_mimetype/add', methods=['POST']) -def content_mimetype_add(): - return encode_data( - get_storage().content_mimetype_add(**decode_request(request))) - - -@app.route('/content_mimetype/missing', methods=['POST']) -def content_mimetype_missing(): - return encode_data( - get_storage().content_mimetype_missing(**decode_request(request))) - - -@app.route('/content_mimetype', methods=['POST']) -def content_mimetype_get(): - return encode_data( - get_storage().content_mimetype_get(**decode_request(request))) - - -@app.route('/content_language/add', methods=['POST']) -def content_language_add(): - return encode_data( - get_storage().content_language_add(**decode_request(request))) - - -@app.route('/content_language/missing', methods=['POST']) -def content_language_missing(): - return encode_data( - get_storage().content_language_missing(**decode_request(request))) - - -@app.route('/content_language', methods=['POST']) -def content_language_get(): - return encode_data( - get_storage().content_language_get(**decode_request(request))) - - -@app.route('/content/ctags/add', methods=['POST']) -def content_ctags_add(): - return encode_data( - get_storage().content_ctags_add(**decode_request(request))) - - -@app.route('/content/ctags/search', methods=['POST']) -def content_ctags_search(): - return encode_data( - get_storage().content_ctags_search(**decode_request(request))) - - -@app.route('/content/ctags/missing', methods=['POST']) -def content_ctags_missing(): - return encode_data( - get_storage().content_ctags_missing(**decode_request(request))) - - -@app.route('/content/ctags', methods=['POST']) -def content_ctags_get(): - return encode_data( - get_storage().content_ctags_get(**decode_request(request))) - - -@app.route('/content/fossology_license/add', methods=['POST']) -def content_fossology_license_add(): - return encode_data( - get_storage().content_fossology_license_add(**decode_request(request))) - - -@app.route('/content/fossology_license', methods=['POST']) -def content_fossology_license_get(): - return encode_data( - get_storage().content_fossology_license_get(**decode_request(request))) - - -@app.route('/indexer_configuration/data', methods=['POST']) -def indexer_configuration_get(): - return encode_data(get_storage().indexer_configuration_get( - **decode_request(request))) - - -@app.route('/indexer_configuration/add', methods=['POST']) -def indexer_configuration_add(): - return encode_data(get_storage().indexer_configuration_add( - **decode_request(request))) - - -@app.route('/content_metadata/add', methods=['POST']) -def content_metadata_add(): - return encode_data( - get_storage().content_metadata_add(**decode_request(request))) - - -@app.route('/content_metadata/missing', methods=['POST']) -def content_metadata_missing(): - return encode_data( - get_storage().content_metadata_missing(**decode_request(request))) - - -@app.route('/content_metadata', methods=['POST']) -def content_metadata_get(): - return encode_data( - get_storage().content_metadata_get(**decode_request(request))) - - -@app.route('/revision_metadata/add', methods=['POST']) -def revision_metadata_add(): - return encode_data( - get_storage().revision_metadata_add(**decode_request(request))) - - -@app.route('/revision_metadata/missing', methods=['POST']) -def revision_metadata_missing(): - return encode_data( - get_storage().revision_metadata_missing(**decode_request(request))) - - -@app.route('/revision_metadata', methods=['POST']) -def revision_metadata_get(): - return encode_data( - get_storage().revision_metadata_get(**decode_request(request))) +# See client.py in the same directory to understand this magic. +for (_name, _meth) in IndexerStorage.__dict__.items(): + if hasattr(_meth, '_endpoint_path'): + def _closure(name, meth): + @app.route('/'+meth._endpoint_path, methods=['POST']) + @functools.wraps(meth) # Copy signature and doc + def _f(): + # Call the actual code + return encode_data( + getattr(get_storage(), name)(**decode_request(request))) + globals()[name] = _f + _closure(_name, _meth) + +del _name, _meth def run_from_webserver(environ, start_response,