diff --git a/swh/indexer/storage/__init__.py b/swh/indexer/storage/__init__.py
--- a/swh/indexer/storage/__init__.py
+++ b/swh/indexer/storage/__init__.py
@@ -19,6 +19,13 @@
INDEXER_CFG_KEY = 'indexer_storage'
+def endpoint(path):
+ def dec(f):
+ f._endpoint_path = path
+ return f
+ return dec
+
+
def get_indexer_storage(cls, args):
"""Get an indexer storage object of class `storage_class` with
arguments `storage_args`.
@@ -72,6 +79,7 @@
return self._db
return Db.from_pool(self._pool)
+ @endpoint('check_config')
def check_config(self, *, check_write):
"""Check that the storage is configured and ready to go."""
# Check permissions on one of the tables
@@ -89,6 +97,7 @@
return True
+ @endpoint('content_mimetype/missing')
@db_transaction_generator()
def content_mimetype_missing(self, mimetypes, db=None, cur=None):
"""List mimetypes missing from storage.
@@ -108,6 +117,7 @@
for obj in db.content_mimetype_missing_from_list(mimetypes, cur):
yield obj[0]
+ @endpoint('content_mimetype/add')
@db_transaction()
def content_mimetype_add(self, mimetypes, conflict_update=False, db=None,
cur=None):
@@ -132,6 +142,7 @@
cur)
db.content_mimetype_add_from_temp(conflict_update, cur)
+ @endpoint('content_mimetype')
@db_transaction_generator()
def content_mimetype_get(self, ids, db=None, cur=None):
"""Retrieve full content mimetype per ids.
@@ -152,6 +163,7 @@
yield converters.db_to_mimetype(
dict(zip(db.content_mimetype_cols, c)))
+ @endpoint('content_language/missing')
@db_transaction_generator()
def content_language_missing(self, languages, db=None, cur=None):
"""List languages missing from storage.
@@ -171,6 +183,7 @@
for obj in db.content_language_missing_from_list(languages, cur):
yield obj[0]
+ @endpoint('content_language')
@db_transaction_generator()
def content_language_get(self, ids, db=None, cur=None):
"""Retrieve full content language per ids.
@@ -190,6 +203,7 @@
yield converters.db_to_language(
dict(zip(db.content_language_cols, c)))
+ @endpoint('content_language/add')
@db_transaction()
def content_language_add(self, languages, conflict_update=False, db=None,
cur=None):
@@ -219,6 +233,7 @@
db.content_language_add_from_temp(conflict_update, cur)
+ @endpoint('content/ctags/missing')
@db_transaction_generator()
def content_ctags_missing(self, ctags, db=None, cur=None):
"""List ctags missing from storage.
@@ -238,6 +253,7 @@
for obj in db.content_ctags_missing_from_list(ctags, cur):
yield obj[0]
+ @endpoint('content/ctags')
@db_transaction_generator()
def content_ctags_get(self, ids, db=None, cur=None):
"""Retrieve ctags per id.
@@ -259,6 +275,7 @@
for c in db.content_ctags_get_from_list(ids, cur):
yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c)))
+ @endpoint('content/ctags/add')
@db_transaction()
def content_ctags_add(self, ctags, conflict_update=False, db=None,
cur=None):
@@ -288,6 +305,7 @@
db.content_ctags_add_from_temp(conflict_update, cur)
+ @endpoint('content/ctags/search')
@db_transaction_generator()
def content_ctags_search(self, expression,
limit=10, last_sha1=None, db=None, cur=None):
@@ -306,6 +324,7 @@
cur=cur):
yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj)))
+ @endpoint('content/fossology_license')
@db_transaction_generator()
def content_fossology_license_get(self, ids, db=None, cur=None):
"""Retrieve licenses per id.
@@ -331,6 +350,7 @@
for id_, facts in d.items():
yield {id_: facts}
+ @endpoint('content/fossology_license/add')
@db_transaction()
def content_fossology_license_add(self, licenses, conflict_update=False,
db=None, cur=None):
@@ -364,6 +384,7 @@
cur=cur)
db.content_fossology_license_add_from_temp(conflict_update, cur)
+ @endpoint('content_metadata/missing')
@db_transaction_generator()
def content_metadata_missing(self, metadata, db=None, cur=None):
"""List metadata missing from storage.
@@ -383,6 +404,7 @@
for obj in db.content_metadata_missing_from_list(metadata, cur):
yield obj[0]
+ @endpoint('content_metadata')
@db_transaction_generator()
def content_metadata_get(self, ids, db=None, cur=None):
"""Retrieve metadata per id.
@@ -402,6 +424,7 @@
yield converters.db_to_metadata(
dict(zip(db.content_metadata_cols, c)))
+ @endpoint('content_metadata/add')
@db_transaction()
def content_metadata_add(self, metadata, conflict_update=False, db=None,
cur=None):
@@ -425,6 +448,7 @@
cur)
db.content_metadata_add_from_temp(conflict_update, cur)
+ @endpoint('revision_metadata/missing')
@db_transaction_generator()
def revision_metadata_missing(self, metadata, db=None, cur=None):
"""List metadata missing from storage.
@@ -443,6 +467,7 @@
for obj in db.revision_metadata_missing_from_list(metadata, cur):
yield obj[0]
+ @endpoint('revision_metadata')
@db_transaction_generator()
def revision_metadata_get(self, ids, db=None, cur=None):
"""Retrieve revision metadata per id.
@@ -462,6 +487,7 @@
yield converters.db_to_metadata(
dict(zip(db.revision_metadata_cols, c)))
+ @endpoint('revision_metadata/add')
@db_transaction()
def revision_metadata_add(self, metadata, conflict_update=False, db=None,
cur=None):
@@ -485,6 +511,7 @@
cur)
db.revision_metadata_add_from_temp(conflict_update, cur)
+ @endpoint('indexer_configuration/add')
@db_transaction_generator()
def indexer_configuration_add(self, tools, db=None, cur=None):
"""Add new tools to the storage.
@@ -513,6 +540,7 @@
for line in tools:
yield dict(zip(db.indexer_configuration_cols, line))
+ @endpoint('indexer_configuration/data')
@db_transaction()
def indexer_configuration_get(self, tool, db=None, cur=None):
"""Retrieve tool information.
diff --git a/swh/indexer/storage/api/client.py b/swh/indexer/storage/api/client.py
--- a/swh/indexer/storage/api/client.py
+++ b/swh/indexer/storage/api/client.py
@@ -3,11 +3,15 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import functools
+import inspect
from swh.core.api import SWHRemoteAPI
from swh.storage.exc import StorageAPIError
+from .. import IndexerStorage
+
class RemoteStorage(SWHRemoteAPI):
"""Proxy to a remote storage API"""
@@ -15,87 +19,30 @@
super().__init__(
api_exception=StorageAPIError, url=url, timeout=timeout)
- def check_config(self, *, check_write):
- return self.post('check_config', {'check_write': check_write})
-
- def content_mimetype_add(self, mimetypes, conflict_update=False):
- return self.post('content_mimetype/add', {
- 'mimetypes': mimetypes,
- 'conflict_update': conflict_update,
- })
-
- def content_mimetype_missing(self, mimetypes):
- return self.post('content_mimetype/missing', {'mimetypes': mimetypes})
-
- def content_mimetype_get(self, ids):
- return self.post('content_mimetype', {'ids': ids})
-
- def content_language_add(self, languages, conflict_update=False):
- return self.post('content_language/add', {
- 'languages': languages,
- 'conflict_update': conflict_update,
- })
-
- def content_language_missing(self, languages):
- return self.post('content_language/missing', {'languages': languages})
-
- def content_language_get(self, ids):
- return self.post('content_language', {'ids': ids})
-
- def content_ctags_add(self, ctags, conflict_update=False):
- return self.post('content/ctags/add', {
- 'ctags': ctags,
- 'conflict_update': conflict_update,
- })
-
- def content_ctags_missing(self, ctags):
- return self.post('content/ctags/missing', {'ctags': ctags})
-
- def content_ctags_get(self, ids):
- return self.post('content/ctags', {'ids': ids})
-
- def content_ctags_search(self, expression, limit=10, last_sha1=None):
- return self.post('content/ctags/search', {
- 'expression': expression,
- 'limit': limit,
- 'last_sha1': last_sha1,
- })
-
- def content_fossology_license_add(self, licenses, conflict_update=False):
- return self.post('content/fossology_license/add', {
- 'licenses': licenses,
- 'conflict_update': conflict_update,
- })
-
- def content_fossology_license_get(self, ids):
- return self.post('content/fossology_license', {'ids': ids})
-
- def content_metadata_add(self, metadata, conflict_update=False):
- return self.post('content_metadata/add', {
- 'metadata': metadata,
- 'conflict_update': conflict_update,
- })
-
- def content_metadata_missing(self, metadata):
- return self.post('content_metadata/missing', {'metadata': metadata})
-
- def content_metadata_get(self, ids):
- return self.post('content_metadata', {'ids': ids})
-
- def revision_metadata_add(self, metadata, conflict_update=False):
- return self.post('revision_metadata/add', {
- 'metadata': metadata,
- 'conflict_update': conflict_update,
- })
-
- def revision_metadata_missing(self, metadata):
- return self.post('revision_metadata/missing', {'metadata': metadata})
-
- def revision_metadata_get(self, ids):
- return self.post('revision_metadata', {'ids': ids})
-
- def indexer_configuration_add(self, tools):
- return self.post('indexer_configuration/add', {'tools': tools})
- def indexer_configuration_get(self, tool):
- return self.post('indexer_configuration/data', {'tool': tool})
+# For each method wrapped with @endpoint in IndexerStorage, add a new
+# method in RemoteStorage, with the same documentation.
+#
+# Note that, despite the usage of decorator magic (eg. functools.wrap),
+# this never actually calls an IndexerStorage method.
+for (_name, _meth) in IndexerStorage.__dict__.items():
+ if hasattr(_meth, '_endpoint_path'):
+ def _closure(name, meth):
+ wrapped_meth = inspect.unwrap(meth)
+
+ @functools.wraps(meth) # Copy signature and doc
+ def _meth(*args, **kwargs):
+ # Match arguments and parameters
+ post_data = inspect.getcallargs(wrapped_meth, *args, **kwargs)
+
+ # Remove arguments that should not be passed
+ self = post_data.pop('self')
+ post_data.pop('cur', None)
+ post_data.pop('db', None)
+
+ # Send the request.
+ return self.post(meth._endpoint_path, post_data)
+ setattr(RemoteStorage, name, _meth)
+ _closure(_name, _meth)
+
+del _name, _meth, _closure
diff --git a/swh/indexer/storage/api/server.py b/swh/indexer/storage/api/server.py
--- a/swh/indexer/storage/api/server.py
+++ b/swh/indexer/storage/api/server.py
@@ -3,6 +3,7 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import functools
import logging
import click
@@ -14,6 +15,8 @@
encode_data_server as encode_data)
from swh.indexer.storage import get_indexer_storage, INDEXER_CFG_KEY
+from .. import IndexerStorage
+
DEFAULT_CONFIG_PATH = 'storage/indexer'
DEFAULT_CONFIG = {
@@ -48,129 +51,20 @@
return 'SWH Indexer Storage API server'
-@app.route('/check_config', methods=['POST'])
-def check_config():
- return encode_data(get_storage().check_config(**decode_request(request)))
-
-
-@app.route('/content_mimetype/add', methods=['POST'])
-def content_mimetype_add():
- return encode_data(
- get_storage().content_mimetype_add(**decode_request(request)))
-
-
-@app.route('/content_mimetype/missing', methods=['POST'])
-def content_mimetype_missing():
- return encode_data(
- get_storage().content_mimetype_missing(**decode_request(request)))
-
-
-@app.route('/content_mimetype', methods=['POST'])
-def content_mimetype_get():
- return encode_data(
- get_storage().content_mimetype_get(**decode_request(request)))
-
-
-@app.route('/content_language/add', methods=['POST'])
-def content_language_add():
- return encode_data(
- get_storage().content_language_add(**decode_request(request)))
-
-
-@app.route('/content_language/missing', methods=['POST'])
-def content_language_missing():
- return encode_data(
- get_storage().content_language_missing(**decode_request(request)))
-
-
-@app.route('/content_language', methods=['POST'])
-def content_language_get():
- return encode_data(
- get_storage().content_language_get(**decode_request(request)))
-
-
-@app.route('/content/ctags/add', methods=['POST'])
-def content_ctags_add():
- return encode_data(
- get_storage().content_ctags_add(**decode_request(request)))
-
-
-@app.route('/content/ctags/search', methods=['POST'])
-def content_ctags_search():
- return encode_data(
- get_storage().content_ctags_search(**decode_request(request)))
-
-
-@app.route('/content/ctags/missing', methods=['POST'])
-def content_ctags_missing():
- return encode_data(
- get_storage().content_ctags_missing(**decode_request(request)))
-
-
-@app.route('/content/ctags', methods=['POST'])
-def content_ctags_get():
- return encode_data(
- get_storage().content_ctags_get(**decode_request(request)))
-
-
-@app.route('/content/fossology_license/add', methods=['POST'])
-def content_fossology_license_add():
- return encode_data(
- get_storage().content_fossology_license_add(**decode_request(request)))
-
-
-@app.route('/content/fossology_license', methods=['POST'])
-def content_fossology_license_get():
- return encode_data(
- get_storage().content_fossology_license_get(**decode_request(request)))
-
-
-@app.route('/indexer_configuration/data', methods=['POST'])
-def indexer_configuration_get():
- return encode_data(get_storage().indexer_configuration_get(
- **decode_request(request)))
-
-
-@app.route('/indexer_configuration/add', methods=['POST'])
-def indexer_configuration_add():
- return encode_data(get_storage().indexer_configuration_add(
- **decode_request(request)))
-
-
-@app.route('/content_metadata/add', methods=['POST'])
-def content_metadata_add():
- return encode_data(
- get_storage().content_metadata_add(**decode_request(request)))
-
-
-@app.route('/content_metadata/missing', methods=['POST'])
-def content_metadata_missing():
- return encode_data(
- get_storage().content_metadata_missing(**decode_request(request)))
-
-
-@app.route('/content_metadata', methods=['POST'])
-def content_metadata_get():
- return encode_data(
- get_storage().content_metadata_get(**decode_request(request)))
-
-
-@app.route('/revision_metadata/add', methods=['POST'])
-def revision_metadata_add():
- return encode_data(
- get_storage().revision_metadata_add(**decode_request(request)))
-
-
-@app.route('/revision_metadata/missing', methods=['POST'])
-def revision_metadata_missing():
- return encode_data(
- get_storage().revision_metadata_missing(**decode_request(request)))
-
-
-@app.route('/revision_metadata', methods=['POST'])
-def revision_metadata_get():
- return encode_data(
- get_storage().revision_metadata_get(**decode_request(request)))
+# See client.py in the same directory to understand this magic.
+for (_name, _meth) in IndexerStorage.__dict__.items():
+ if hasattr(_meth, '_endpoint_path'):
+ def _closure(name, meth):
+ @app.route('/'+meth._endpoint_path, methods=['POST'])
+ @functools.wraps(meth) # Copy signature and doc
+ def _f():
+ # Call the actual code
+ return encode_data(
+ getattr(get_storage(), name)(**decode_request(request)))
+ globals()[name] = _f
+ _closure(_name, _meth)
+
+del _name, _meth
def run_from_webserver(environ, start_response,