diff --git a/PKG-INFO b/PKG-INFO index 285aac1d..01d808da 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.storage -Version: 0.0.20 +Version: 0.0.21 Summary: Software Heritage storage manager Home-page: https://forge.softwareheritage.org/diffusion/DSTO/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh.storage.egg-info/PKG-INFO b/swh.storage.egg-info/PKG-INFO index 285aac1d..01d808da 100644 --- a/swh.storage.egg-info/PKG-INFO +++ b/swh.storage.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.storage -Version: 0.0.20 +Version: 0.0.21 Summary: Software Heritage storage manager Home-page: https://forge.softwareheritage.org/diffusion/DSTO/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh/storage/__init__.py b/swh/storage/__init__.py index 4b870c3f..3582630c 100644 --- a/swh/storage/__init__.py +++ b/swh/storage/__init__.py @@ -1,3 +1,27 @@ from . import storage Storage = storage.Storage + + +def get_storage(storage_class, storage_args): + """ + Get a storage object of class `storage_class` with arguments + `storage_args`. + + Args: + storage_class: one of 'remote_storage', 'local_storage' + storage_args: the arguments to pass to the storage class + Returns: + an instance of swh.storage.Storage (either local or remote) + Raises: + ValueError if passed an unknown storage_class. + """ + + if storage_class == 'remote_storage': + from .storage.api.client import RemoteStorage as Storage + elif storage_class == 'local_storage': + from .storage import Storage + else: + raise ValueError('Unknown storage class `%s`' % storage_class) + + return Storage(*storage_args) diff --git a/swh/storage/api/client.py b/swh/storage/api/client.py index cbeb78c8..c684d4b0 100644 --- a/swh/storage/api/client.py +++ b/swh/storage/api/client.py @@ -1,148 +1,152 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pickle import requests from swh.core.serializers import msgpack_dumps, msgpack_loads, SWHJSONDecoder def encode_data(data): - return msgpack_dumps(data) + try: + return msgpack_dumps(data) + except OverflowError as e: + raise ValueError('Limits were reached. Please, check your input.\n' + + str(e)) def decode_response(response): content_type = response.headers['content-type'] if content_type.startswith('application/x-msgpack'): r = msgpack_loads(response.content) elif content_type.startswith('application/json'): r = response.json(cls=SWHJSONDecoder) else: raise ValueError('Wrong content type `%s` for API response' % content_type) return r class RemoteStorage(): """Proxy to a remote storage API""" def __init__(self, base_url): self.base_url = base_url self.session = requests.Session() def url(self, endpoint): return '%s%s' % (self.base_url, endpoint) def post(self, endpoint, data): response = self.session.post( self.url(endpoint), data=encode_data(data), headers={'content-type': 'application/x-msgpack'}, ) # XXX: this breaks language-independence and should be # replaced by proper unserialization if response.status_code == 400: raise pickle.loads(decode_response(response)) return decode_response(response) def get(self, endpoint, data=None): response = self.session.get( self.url(endpoint), params=data, ) if response.status_code == 404: return None # XXX: this breaks language-independence and should be # replaced by proper unserialization if response.status_code == 400: raise pickle.loads(decode_response(response)) else: return decode_response(response) def content_add(self, content): return self.post('content/add', {'content': content}) def content_missing(self, content, key_hash='sha1'): return self.post('content/missing', {'content': content, 'key_hash': key_hash}) def content_get(self, content): return self.post('content/data', {'content': content}) def content_find(self, content): return self.post('content/present', {'content': content}) def content_find_occurrence(self, content): return self.post('content/occurrence', {'content': content}) def directory_add(self, directories): return self.post('directory/add', {'directories': directories}) def directory_missing(self, directories): return self.post('directory/missing', {'directories': directories}) def directory_get(self, directory, recursive=False): return self.get('directory', {'directory': directory, 'recursive': recursive}) def revision_get(self, revisions): return self.post('revision', {'revisions': revisions}) def revision_log(self, revisions): return self.post('revision/log', {'revisions': revisions}) def revision_add(self, revisions): return self.post('revision/add', {'revisions': revisions}) def revision_missing(self, revisions): return self.post('revision/missing', {'revisions': revisions}) def release_add(self, releases): return self.post('release/add', {'releases': releases}) def release_get(self, releases): return self.post('release', {'releases': releases}) def release_missing(self, releases): return self.post('release/missing', {'releases': releases}) def occurrence_add(self, occurrences): return self.post('occurrence/add', {'occurrences': occurrences}) def origin_get(self, origin): return self.post('origin/get', {'origin': origin}) def origin_add_one(self, origin): return self.post('origin/add', {'origin': origin}) def person_get(self, person): return self.post('person', {'person': person}) def fetch_history_start(self, origin_id): return self.post('fetch_history/start', {'origin_id': origin_id}) def fetch_history_end(self, fetch_history_id, data): return self.post('fetch_history/end', {'fetch_history_id': fetch_history_id, 'data': data}) def fetch_history_get(self, fetch_history_id): return self.get('fetch_history', {'id': fetch_history_id}) def entity_add(self, entities): return self.post('entity/add', {'entities': entities}) def entity_get_from_lister_metadata(self, entities): return self.post('entity/from_lister_metadata', {'entities': entities}) def stat_counters(self): return self.get('stat/counters') diff --git a/version.txt b/version.txt index b9e1ad5c..f37854f0 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.20-0-g691e24a \ No newline at end of file +v0.0.21-0-g1938416 \ No newline at end of file