diff --git a/swh/storage/api/server.py b/swh/storage/api/server.py --- a/swh/storage/api/server.py +++ b/swh/storage/api/server.py @@ -21,6 +21,11 @@ storage = None +OBJECT_COUNTER_MAPPING = { + 'content_add': 'new', # ['new', 'new-missing'], +} + + def timed(f): """Time that function! @@ -34,12 +39,42 @@ return d +def encode(f): + @wraps(f) + def d(*a, **kw): + r = f(*a, **kw) + return encode_data(r) + + return d + + +def increment(f): + """Increment object counters for the decorated function. + + """ + @wraps(f) + def d(*a, **kw): + # execute the function + r = f(*a, **kw) + + # extract the metric information from the summary result + counter_key = OBJECT_COUNTER_MAPPING.get(f.__name__) + if counter_key: + value = r.get(counter_key) + if value: + statsd.increment('swh_storage_request_object_count', + value, tags={'endpoint': f.__name__}) + + return r + + return d + + @app.errorhandler(Exception) def my_error_handler(exception): return error_handler(exception, encode_data) -@timed def get_storage(): global storage if not storage: @@ -91,8 +126,10 @@ @app.route('/content/add', methods=['POST']) @timed +@encode +@increment def content_add(): - return encode_data(get_storage().content_add(**decode_request(request))) + return get_storage().content_add(**decode_request(request)) @app.route('/content/update', methods=['POST']) diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -115,6 +115,9 @@ content in """ + summary = { + 'all': len(content), + } if self.journal_writer: for item in content: if 'data' in item: @@ -188,6 +191,8 @@ else: raise + summary['new'] = len(missing_content) + if missing_skipped: missing_filtered = ( cont for cont in content_without_data @@ -200,11 +205,14 @@ # move metadata in place db.skipped_content_add_from_temp(cur) + summary['new-skipped'] = len(missing_skipped) # Wait for objstorage addition before returning from the # transaction, bubbling up any exception added_to_objstorage.result() + return summary + @db_transaction() def content_update(self, content, keys=[], db=None, cur=None): """Update content blobs to the storage. Does nothing for unknown