diff --git a/.gitignore b/.gitignore index f5fc2ae..95c4f7c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ *.pyc *.sw? *~ .coverage .eggs/ __pycache__ *.egg-info/ -version.txt \ No newline at end of file +build +dist +version.txt diff --git a/PKG-INFO b/PKG-INFO index dc48822..915a85d 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,10 +1,24 @@ -Metadata-Version: 1.0 +Metadata-Version: 2.1 Name: swh.objstorage -Version: 0.0.26 +Version: 0.0.27 Summary: Software Heritage Object Storage Home-page: https://forge.softwareheritage.org/diffusion/DOBJS Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN -Description: UNKNOWN +Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest +Project-URL: Funding, https://www.softwareheritage.org/donate +Project-URL: Source, https://forge.softwareheritage.org/source/swh-objstorage +Description: swh-objstorage + ============== + + Content-addressable object storage for the Software Heritage project. + Platform: UNKNOWN +Classifier: Programming Language :: Python :: 3 +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) +Classifier: Operating System :: OS Independent +Classifier: Development Status :: 5 - Production/Stable +Description-Content-Type: text/markdown +Provides-Extra: testing diff --git a/README.md b/README.md new file mode 100644 index 0000000..d74885a --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +swh-objstorage +============== + +Content-addressable object storage for the Software Heritage project. diff --git a/bin/swh-objstorage-add-dir b/bin/swh-objstorage-add-dir index c1dd69d..59bdabf 100755 --- a/bin/swh-objstorage-add-dir +++ b/bin/swh-objstorage-add-dir @@ -1,37 +1,37 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import os import sys from swh.storage import objstorage if __name__ == '__main__': try: root_dir = sys.argv[1] dirname = sys.argv[2] except IndexError: print("Usage: swh-objstorage-add-dir OBJ_STORAGE_DIR DATA_DIR") sys.exit(1) logging.basicConfig(level=logging.INFO) objs = objstorage.ObjStorage(root_dir) dups = 0 for root, _dirs, files in os.walk(dirname): for name in files: path = os.path.join(root, name) with open(path, 'rb') as f: try: objs.add(f.read()) except objstorage.DuplicateObjError: dups += 1 if dups: logging.info('skipped %d duplicate(s) file(s)' % dups) diff --git a/bin/swh-objstorage-fsck b/bin/swh-objstorage-fsck index b277883..67f3851 100755 --- a/bin/swh-objstorage-fsck +++ b/bin/swh-objstorage-fsck @@ -1,28 +1,28 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import sys from swh.storage import objstorage if __name__ == '__main__': try: root_dir = sys.argv[1] except IndexError: print("Usage: swh-objstorage-add-dir OBJ_STORAGE_DIR") sys.exit(1) logging.basicConfig(level=logging.INFO) objs = objstorage.ObjStorage(root_dir) for obj_id in objs: try: objs.check(obj_id) except objstorage.Error as err: logging.error(err) diff --git a/docs/index.rst b/docs/index.rst index 77b43fa..912be4a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,17 +1,19 @@ .. _swh-objstorage: -Software Heritage - Development Documentation -============================================= +Software Heritage - Object storage +================================== + +Content-addressable object storage. + .. toctree:: :maxdepth: 2 :caption: Contents: - Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` diff --git a/requirements-swh.txt b/requirements-swh.txt index c193587..24f6e64 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,2 +1,2 @@ -swh.core >= 0.0.37 +swh.core >= 0.0.41 swh.model >= 0.0.14 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..8ae56d8 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,4 @@ +nose +apache-libcloud +azure-storage +python-cephlibs diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index daf3104..5fd9abc --- a/setup.py +++ b/setup.py @@ -1,33 +1,68 @@ +#!/usr/bin/env python3 +# Copyright (C) 2015-2018 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + from setuptools import setup, find_packages +from os import path +from io import open + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the README file +with open(path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + + +def parse_requirements(name=None): + if name: + reqf = 'requirements-%s.txt' % name + else: + reqf = 'requirements.txt' -def parse_requirements(): requirements = [] - for reqf in ('requirements.txt', 'requirements-swh.txt'): - with open(reqf) as f: - for line in f.readlines(): - line = line.strip() - if not line or line.startswith('#'): - continue - requirements.append(line) + if not path.exists(reqf): + return requirements + + with open(reqf) as f: + for line in f.readlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + requirements.append(line) return requirements -# Edit this part to match your module -# full sample: https://forge.softwareheritage.org/diffusion/DCORE/browse/master/setup.py setup( name='swh.objstorage', description='Software Heritage Object Storage', + long_description=long_description, + long_description_content_type='text/markdown', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DOBJS', packages=find_packages(), scripts=[ 'bin/swh-objstorage-add-dir', 'bin/swh-objstorage-fsck' ], # scripts to package - install_requires=parse_requirements(), + install_requires=parse_requirements() + parse_requirements('swh'), setup_requires=['vcversioner'], + extras_require={'testing': parse_requirements('test')}, vcversioner={}, include_package_data=True, + classifiers=[ + "Programming Language :: Python :: 3", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Operating System :: OS Independent", + "Development Status :: 5 - Production/Stable", + ], + project_urls={ + 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', + 'Funding': 'https://www.softwareheritage.org/donate', + 'Source': 'https://forge.softwareheritage.org/source/swh-objstorage', + }, ) diff --git a/swh.objstorage.egg-info/PKG-INFO b/swh.objstorage.egg-info/PKG-INFO index dc48822..915a85d 100644 --- a/swh.objstorage.egg-info/PKG-INFO +++ b/swh.objstorage.egg-info/PKG-INFO @@ -1,10 +1,24 @@ -Metadata-Version: 1.0 +Metadata-Version: 2.1 Name: swh.objstorage -Version: 0.0.26 +Version: 0.0.27 Summary: Software Heritage Object Storage Home-page: https://forge.softwareheritage.org/diffusion/DOBJS Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN -Description: UNKNOWN +Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest +Project-URL: Funding, https://www.softwareheritage.org/donate +Project-URL: Source, https://forge.softwareheritage.org/source/swh-objstorage +Description: swh-objstorage + ============== + + Content-addressable object storage for the Software Heritage project. + Platform: UNKNOWN +Classifier: Programming Language :: Python :: 3 +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) +Classifier: Operating System :: OS Independent +Classifier: Development Status :: 5 - Production/Stable +Description-Content-Type: text/markdown +Provides-Extra: testing diff --git a/swh.objstorage.egg-info/SOURCES.txt b/swh.objstorage.egg-info/SOURCES.txt index e62dbd6..178a587 100644 --- a/swh.objstorage.egg-info/SOURCES.txt +++ b/swh.objstorage.egg-info/SOURCES.txt @@ -1,60 +1,62 @@ .gitignore AUTHORS LICENSE MANIFEST.in Makefile +README.md requirements-swh.txt +requirements-test.txt requirements.txt setup.py version.txt bin/swh-objstorage-add-dir bin/swh-objstorage-azure bin/swh-objstorage-fsck debian/changelog debian/compat debian/control debian/copyright debian/rules debian/source/format docs/.gitignore docs/Makefile docs/conf.py docs/index.rst docs/_static/.placeholder docs/_templates/.placeholder swh/__init__.py swh.objstorage.egg-info/PKG-INFO swh.objstorage.egg-info/SOURCES.txt swh.objstorage.egg-info/dependency_links.txt swh.objstorage.egg-info/requires.txt swh.objstorage.egg-info/top_level.txt swh/objstorage/__init__.py swh/objstorage/exc.py swh/objstorage/objstorage.py swh/objstorage/objstorage_in_memory.py swh/objstorage/objstorage_pathslicing.py swh/objstorage/objstorage_rados.py swh/objstorage/api/__init__.py swh/objstorage/api/client.py swh/objstorage/api/server.py swh/objstorage/cloud/__init__.py swh/objstorage/cloud/objstorage_azure.py swh/objstorage/cloud/objstorage_cloud.py swh/objstorage/multiplexer/__init__.py swh/objstorage/multiplexer/multiplexer_objstorage.py swh/objstorage/multiplexer/striping_objstorage.py swh/objstorage/multiplexer/filter/__init__.py swh/objstorage/multiplexer/filter/filter.py swh/objstorage/multiplexer/filter/id_filter.py swh/objstorage/multiplexer/filter/read_write_filter.py swh/objstorage/tests/__init__.py swh/objstorage/tests/objstorage_testing.py swh/objstorage/tests/test_multiplexer_filter.py swh/objstorage/tests/test_objstorage_api.py swh/objstorage/tests/test_objstorage_azure.py swh/objstorage/tests/test_objstorage_cloud.py swh/objstorage/tests/test_objstorage_in_memory.py swh/objstorage/tests/test_objstorage_instantiation.py swh/objstorage/tests/test_objstorage_multiplexer.py swh/objstorage/tests/test_objstorage_pathslicing.py swh/objstorage/tests/test_objstorage_striping.py \ No newline at end of file diff --git a/swh.objstorage.egg-info/requires.txt b/swh.objstorage.egg-info/requires.txt index eeba3e8..dd012b4 100644 --- a/swh.objstorage.egg-info/requires.txt +++ b/swh.objstorage.egg-info/requires.txt @@ -1,5 +1,11 @@ aiohttp>=2.1.0 click -swh.core>=0.0.37 +swh.core>=0.0.41 swh.model>=0.0.14 vcversioner + +[testing] +apache-libcloud +azure-storage +nose +python-cephlibs diff --git a/swh/objstorage/__init__.py b/swh/objstorage/__init__.py index fd36149..03c6d5a 100644 --- a/swh/objstorage/__init__.py +++ b/swh/objstorage/__init__.py @@ -1,86 +1,92 @@ # Copyright (C) 2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from .objstorage import ObjStorage from .objstorage_pathslicing import PathSlicingObjStorage from .objstorage_in_memory import InMemoryObjStorage from .api.client import RemoteObjStorage from .multiplexer import MultiplexerObjStorage, StripingObjStorage from .multiplexer.filter import add_filters __all__ = ['get_objstorage', 'ObjStorage'] _STORAGE_CLASSES = { 'pathslicing': PathSlicingObjStorage, 'remote': RemoteObjStorage, 'in-memory': InMemoryObjStorage, } +_STORAGE_CLASSES_MISSING = { +} + try: from swh.objstorage.cloud.objstorage_azure import ( AzureCloudObjStorage, PrefixedAzureCloudObjStorage, ) _STORAGE_CLASSES['azure'] = AzureCloudObjStorage _STORAGE_CLASSES['azure-prefixed'] = PrefixedAzureCloudObjStorage -except ImportError: - pass +except ImportError as e: + _STORAGE_CLASSES_MISSING['azure'] = e.args[0] + _STORAGE_CLASSES_MISSING['azure-prefixed'] = e.args[0] try: from swh.objstorage.objstorage_rados import RADOSObjStorage _STORAGE_CLASSES['rados'] = RADOSObjStorage -except ImportError: - pass +except ImportError as e: + _STORAGE_CLASSES_MISSING['rados'] = e.args[0] def get_objstorage(cls, args): """ Create an ObjStorage using the given implementation class. Args: cls (str): objstorage class unique key contained in the _STORAGE_CLASSES dict. args (dict): arguments for the required class of objstorage that must match exactly the one in the `__init__` method of the class. Returns: subclass of ObjStorage that match the given `storage_class` argument. Raises: ValueError: if the given storage class is not a valid objstorage key. """ - try: + if cls in _STORAGE_CLASSES: return _STORAGE_CLASSES[cls](**args) - except KeyError: - raise ValueError('Storage class %s does not exist' % cls) + else: + raise ValueError('Storage class {} is not available: {}'.format( + cls, + _STORAGE_CLASSES_MISSING.get(cls, 'unknown name'))) def _construct_filtered_objstorage(storage_conf, filters_conf): return add_filters( get_objstorage(**storage_conf), filters_conf ) _STORAGE_CLASSES['filtered'] = _construct_filtered_objstorage def _construct_multiplexer_objstorage(objstorages): storages = [get_objstorage(**conf) for conf in objstorages] return MultiplexerObjStorage(storages) _STORAGE_CLASSES['multiplexer'] = _construct_multiplexer_objstorage def _construct_striping_objstorage(objstorages): storages = [get_objstorage(**conf) for conf in objstorages] return StripingObjStorage(storages) _STORAGE_CLASSES['striping'] = _construct_striping_objstorage diff --git a/swh/objstorage/api/client.py b/swh/objstorage/api/client.py index d0116ae..7c6a7cf 100644 --- a/swh/objstorage/api/client.py +++ b/swh/objstorage/api/client.py @@ -1,78 +1,87 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import abc -from swh.core.api import SWHRemoteAPI +from swh.core.api import SWHRemoteAPI, MetaSWHRemoteAPI from swh.model import hashutil from ..objstorage import ObjStorage, DEFAULT_CHUNK_SIZE from ..exc import ObjNotFoundError, ObjStorageAPIError -class RemoteObjStorage(ObjStorage, SWHRemoteAPI): +class MetaRemoteObjStorage(MetaSWHRemoteAPI, abc.ABCMeta): + """Hackish class to make multiple inheritance with different metaclasses + work.""" + pass + + +class RemoteObjStorage(ObjStorage, SWHRemoteAPI, + metaclass=MetaRemoteObjStorage): """Proxy to a remote object storage. This class allows to connect to an object storage server via http protocol. Attributes: url (string): The url of the server to connect. Must end with a '/' session: The session to send requests. """ + def __init__(self, url, **kwargs): super().__init__(api_exception=ObjStorageAPIError, url=url, **kwargs) def check_config(self, *, check_write): return self.post('check_config', {'check_write': check_write}) def __contains__(self, obj_id): return self.post('content/contains', {'obj_id': obj_id}) def add(self, content, obj_id=None, check_presence=True): return self.post('content/add', {'content': content, 'obj_id': obj_id, 'check_presence': check_presence}) def add_batch(self, contents, check_presence=True): return self.post('content/add/batch', { 'contents': contents, 'check_presence': check_presence, }) def get(self, obj_id): ret = self.post('content/get', {'obj_id': obj_id}) if ret is None: raise ObjNotFoundError(obj_id) else: return ret def get_batch(self, obj_ids): return self.post('content/get/batch', {'obj_ids': obj_ids}) def check(self, obj_id): return self.post('content/check', {'obj_id': obj_id}) def delete(self, obj_id): super().delete(obj_id) # Check delete permission return self.post('content/delete', {'obj_id': obj_id}) # Management methods def get_random(self, batch_size): return self.post('content/get/random', {'batch_size': batch_size}) # Streaming methods def add_stream(self, content_iter, obj_id, check_presence=True): obj_id = hashutil.hash_to_hex(obj_id) return self.post_stream('content/add_stream/{}'.format(obj_id), params={'check_presence': check_presence}, data=content_iter) def get_stream(self, obj_id, chunk_size=DEFAULT_CHUNK_SIZE): obj_id = hashutil.hash_to_hex(obj_id) return super().get_stream('content/get_stream/{}'.format(obj_id), chunk_size=chunk_size) diff --git a/swh/objstorage/cloud/objstorage_cloud.py b/swh/objstorage/cloud/objstorage_cloud.py index 8778f7e..ef78b18 100644 --- a/swh/objstorage/cloud/objstorage_cloud.py +++ b/swh/objstorage/cloud/objstorage_cloud.py @@ -1,169 +1,169 @@ # Copyright (C) 2016-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import abc from swh.model import hashutil from swh.objstorage.objstorage import ObjStorage, compute_hash from swh.objstorage.exc import ObjNotFoundError, Error from libcloud.storage import providers from libcloud.storage.types import Provider, ObjectDoesNotExistError class CloudObjStorage(ObjStorage, metaclass=abc.ABCMeta): """Abstract ObjStorage that connect to a cloud using Libcloud Implementations of this class must redefine the _get_provider method to make it return a driver provider (i.e. object that supports `get_driver` method) which return a LibCloud driver (see https://libcloud.readthedocs.io/en/latest/storage/api.html). """ def __init__(self, api_key, api_secret_key, container_name, **kwargs): super().__init__(**kwargs) self.driver = self._get_driver(api_key, api_secret_key) self.container_name = container_name self.container = self.driver.get_container( container_name=container_name) def _get_driver(self, api_key, api_secret_key): """Initialize a driver to communicate with the cloud Args: api_key: key to connect to the API. - api_secret_key: secret key for authentification. + api_secret_key: secret key for authentication. Returns: a Libcloud driver to a cloud storage. """ # Get the driver class from its description. cls = providers.get_driver(self._get_provider()) # Initialize the driver. return cls(api_key, api_secret_key) @abc.abstractmethod def _get_provider(self): """Get a libcloud driver provider - This method must be overriden by subclasses to specify which + This method must be overridden by subclasses to specify which of the native libcloud driver the current storage should connect to. Alternatively, provider for a custom driver may be returned, in which case the provider will have to support `get_driver` method. """ raise NotImplementedError('%s must implement `get_provider` method' % type(self)) def check_config(self, *, check_write): """Check the configuration for this object storage""" # FIXME: hopefully this blew up during instantiation return True def __contains__(self, obj_id): try: self._get_object(obj_id) except ObjNotFoundError: return False else: return True def __iter__(self): """ Iterate over the objects present in the storage Warning: Iteration over the contents of a cloud-based object storage may have bad efficiency: due to the very high amount of objects in it and the fact that it is remote, get all the contents of the current object storage may result in a lot of network requests. You almost certainly don't want to use this method in production. """ yield from map(lambda obj: obj.name, self.driver.iterate_container_objects(self.container)) def __len__(self): """Compute the number of objects in the current object storage. Warning: this currently uses `__iter__`, its warning about bad performance applies. Returns: number of objects contained in the storage. """ return sum(1 for i in self) def add(self, content, obj_id=None, check_presence=True): if obj_id is None: # Checksum is missing, compute it on the fly. obj_id = compute_hash(content) if check_presence and obj_id in self: return obj_id self._put_object(content, obj_id) return obj_id def restore(self, content, obj_id=None): return self.add(content, obj_id, check_presence=False) def get(self, obj_id): return bytes(self._get_object(obj_id).as_stream()) def check(self, obj_id): # Check that the file exists, as _get_object raises ObjNotFoundError self._get_object(obj_id) # Check the content integrity obj_content = self.get(obj_id) content_obj_id = compute_hash(obj_content) if content_obj_id != obj_id: raise Error(obj_id) def delete(self, obj_id): super().delete(obj_id) # Check delete permission obj = self._get_object(obj_id) return self.driver.delete_object(obj) def _get_object(self, obj_id): """Get a Libcloud wrapper for an object pointer. This wrapper does not retrieve the content of the object directly. """ hex_obj_id = hashutil.hash_to_hex(obj_id) try: return self.driver.get_object(self.container_name, hex_obj_id) except ObjectDoesNotExistError as e: raise ObjNotFoundError(obj_id) def _put_object(self, content, obj_id): """Create an object in the cloud storage. Created object will contain the content and be referenced by the given id. """ hex_obj_id = hashutil.hash_to_hex(obj_id) self.driver.upload_object_via_stream(iter(content), self.container, hex_obj_id) class AwsCloudObjStorage(CloudObjStorage): """ Amazon's S3 Cloud-based object storage """ def _get_provider(self): return Provider.S3 class OpenStackCloudObjStorage(CloudObjStorage): """ OpenStack Swift Cloud based object storage """ def _get_provider(self): return Provider.OPENSTACK_SWIFT diff --git a/swh/objstorage/multiplexer/multiplexer_objstorage.py b/swh/objstorage/multiplexer/multiplexer_objstorage.py index 65b00e4..6b17bb9 100644 --- a/swh/objstorage/multiplexer/multiplexer_objstorage.py +++ b/swh/objstorage/multiplexer/multiplexer_objstorage.py @@ -1,299 +1,299 @@ # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import queue import random import threading from ..objstorage import ObjStorage from ..exc import ObjNotFoundError class ObjStorageThread(threading.Thread): def __init__(self, storage): super().__init__(daemon=True) self.storage = storage self.commands = queue.Queue() def run(self): while True: try: mailbox, command, args, kwargs = self.commands.get(True, 0.05) except queue.Empty: continue try: ret = getattr(self.storage, command)(*args, **kwargs) except Exception as exc: self.queue_result(mailbox, 'exception', exc) else: self.queue_result(mailbox, 'result', ret) - def queue_command(self, command, *args, mailbox=None, **kwargs): """Enqueue a new command to be processed by the thread. Args: command (str): one of the method names for the underlying storage. mailbox (queue.Queue): explicit mailbox if the calling thread wants to override it. args, kwargs: arguments for the command. Returns: queue.Queue The mailbox you can read the response from """ if not mailbox: mailbox = queue.Queue() self.commands.put((mailbox, command, args, kwargs)) return mailbox def queue_result(self, mailbox, result_type, result): """Enqueue a new result in the mailbox This also provides a reference to the storage, which can be useful when an exceptional condition arises. Args: mailbox (queue.Queue): the mailbox to which we need to enqueue the result result_type (str): one of 'result', 'exception' result: the result to pass back to the calling thread """ mailbox.put({ 'type': result_type, 'result': result, }) @staticmethod def get_result_from_mailbox(mailbox, *args, **kwargs): """Unpack the result from the mailbox. Arguments: mailbox (queue.Queue): A mailbox to unpack a result from args, kwargs: arguments to :func:`mailbox.get` Returns: the next result unpacked from the queue Raises: either the exception we got back from the underlying storage, or :exc:`queue.Empty` if :func:`mailbox.get` raises that. """ result = mailbox.get(*args, **kwargs) if result['type'] == 'exception': raise result['result'] from None else: return result['result'] @staticmethod def collect_results(mailbox, num_results): """Collect num_results from the mailbox""" collected = 0 ret = [] while collected < num_results: try: ret.append(ObjStorageThread.get_result_from_mailbox( mailbox, True, 0.05 )) except queue.Empty: continue collected += 1 return ret def __getattr__(self, attr): def call(*args, **kwargs): mailbox = self.queue_command(attr, *args, **kwargs) return self.get_result_from_mailbox(mailbox) return call def __contains__(self, *args, **kwargs): mailbox = self.queue_command('__contains__', *args, **kwargs) return self.get_result_from_mailbox(mailbox) class MultiplexerObjStorage(ObjStorage): """Implementation of ObjStorage that distributes between multiple storages. The multiplexer object storage allows an input to be demultiplexed among multiple storages that will or will not accept it by themselves (see .filter package). - As the ids can be differents, no pre-computed ids should be + As the ids can be different, no pre-computed ids should be submitted. Also, there are no guarantees that the returned ids can be used directly into the storages that the multiplexer manage. Use case examples follow. Example 1:: storage_v1 = filter.read_only(PathSlicingObjStorage('/dir1', '0:2/2:4/4:6')) storage_v2 = PathSlicingObjStorage('/dir2', '0:1/0:5') storage = MultiplexerObjStorage([storage_v1, storage_v2]) When using 'storage', all the new contents will only be added to the v2 storage, while it will be retrievable from both. Example 2:: storage_v1 = filter.id_regex( PathSlicingObjStorage('/dir1', '0:2/2:4/4:6'), r'[^012].*' ) storage_v2 = filter.if_regex( PathSlicingObjStorage('/dir2', '0:1/0:5'), r'[012]/*' ) storage = MultiplexerObjStorage([storage_v1, storage_v2]) When using this storage, the contents with a sha1 starting with 0, 1 or 2 will be redirected (read AND write) to the storage_v2, while the others will be redirected to the storage_v1. If a content starting with 0, 1 or 2 is present in the storage_v1, it would be ignored anyway. """ def __init__(self, storages, **kwargs): super().__init__(**kwargs) self.storages = storages self.storage_threads = [ ObjStorageThread(storage) for storage in storages ] for thread in self.storage_threads: thread.start() def wrap_call(self, threads, call, *args, **kwargs): threads = list(threads) mailbox = queue.Queue() for thread in threads: thread.queue_command(call, *args, mailbox=mailbox, **kwargs) return ObjStorageThread.collect_results(mailbox, len(threads)) def get_read_threads(self, obj_id=None): yield from self.storage_threads def get_write_threads(self, obj_id=None): yield from self.storage_threads def check_config(self, *, check_write): """Check whether the object storage is properly configured. Args: check_write (bool): if True, check if writes to the object storage can succeed. Returns: True if the configuration check worked, an exception if it didn't. """ return all( self.wrap_call(self.storage_threads, 'check_config', check_write=check_write) ) def __contains__(self, obj_id): """Indicate if the given object is present in the storage. Args: obj_id (bytes): object identifier. Returns: - True iff the object is present in the current object storage. + True if and only if the object is present in the current object + storage. """ for storage in self.get_read_threads(obj_id): if obj_id in storage: return True return False def add(self, content, obj_id=None, check_presence=True): """ Add a new object to the object storage. If the adding step works in all the storages that accept this content, this is a success. Otherwise, the full adding step is an error even if it succeed in some of the storages. Args: content: content of the object to be added to the storage. obj_id: checksum of [bytes] using [ID_HASH_ALGO] algorithm. When given, obj_id will be trusted to match the bytes. If missing, obj_id will be computed on the fly. check_presence: indicate if the presence of the content should be verified before adding the file. Returns: an id of the object into the storage. As the write-storages are always readable as well, any id will be valid to retrieve a content. """ return self.wrap_call( self.get_write_threads(obj_id), 'add', content, obj_id=obj_id, check_presence=check_presence, ).pop() def add_batch(self, contents, check_presence=True): """Add a batch of new objects to the object storage. """ write_threads = list(self.get_write_threads()) return sum(self.wrap_call( write_threads, 'add_batch', contents, check_presence=check_presence, )) // len(write_threads) def restore(self, content, obj_id=None): return self.wrap_call( self.get_write_threads(obj_id), 'restore', content, obj_id=obj_id, ).pop() def get(self, obj_id): for storage in self.get_read_threads(obj_id): try: return storage.get(obj_id) except ObjNotFoundError: continue # If no storage contains this content, raise the error raise ObjNotFoundError(obj_id) def check(self, obj_id): nb_present = 0 for storage in self.get_read_threads(obj_id): try: storage.check(obj_id) except ObjNotFoundError: continue else: nb_present += 1 # If there is an Error because of a corrupted file, then let it pass. # Raise the ObjNotFoundError only if the content couldn't be found in # all the storages. if nb_present == 0: raise ObjNotFoundError(obj_id) def delete(self, obj_id): super().delete(obj_id) # Check delete permission return all( self.wrap_call(self.get_write_threads(obj_id), 'delete', obj_id) ) def get_random(self, batch_size): storages_set = [storage for storage in self.storages if len(storage) > 0] if len(storages_set) <= 0: return [] while storages_set: storage = random.choice(storages_set) try: return storage.get_random(batch_size) except NotImplementedError: storages_set.remove(storage) # There is no storage that allow the get_random operation raise NotImplementedError( "There is no storage implementation into the multiplexer that " "support the 'get_random' operation" ) diff --git a/swh/objstorage/objstorage.py b/swh/objstorage/objstorage.py index 60ed817..b72214d 100644 --- a/swh/objstorage/objstorage.py +++ b/swh/objstorage/objstorage.py @@ -1,277 +1,278 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import abc from swh.model import hashutil from .exc import ObjNotFoundError ID_HASH_ALGO = 'sha1' ID_HASH_LENGTH = 40 # Size in bytes of the hash hexadecimal representation. DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024 # Size in bytes of the streaming chunks def compute_hash(content): return hashutil.hash_data( content, algorithms=[ID_HASH_ALGO] ).get(ID_HASH_ALGO) class ObjStorage(metaclass=abc.ABCMeta): """ High-level API to manipulate the Software Heritage object storage. Conceptually, the object storage offers the following methods: - check_config() check if the object storage is properly configured - __contains__() check if an object is present, by object id - add() add a new object, returning an object id - restore() same as add() but erase an already existed content - get() retrieve the content of an object, by object id - check() check the integrity of an object, by object id - delete() remove an object And some management methods: - get_random() get random object id of existing contents (used for the content integrity checker). Some of the methods have available streaming equivalents: - add_stream() same as add() but with a chunked iterator - restore_stream() same as add_stream() but erase already existing content - get_stream() same as get() but returns a chunked iterator Each implementation of this interface can have a different behavior and its own way to store the contents. """ def __init__(self, *, allow_delete=False, **kwargs): # A more complete permission system could be used in place of that if # it becomes needed super().__init__(**kwargs) self.allow_delete = allow_delete @abc.abstractmethod def check_config(self, *, check_write): """Check whether the object storage is properly configured. Args: check_write (bool): if True, check if writes to the object storage can succeed. Returns: True if the configuration check worked, an exception if it didn't. """ pass @abc.abstractmethod def __contains__(self, obj_id, *args, **kwargs): """Indicate if the given object is present in the storage. Args: obj_id (bytes): object identifier. Returns: - True iff the object is present in the current object storage. + True if and only if the object is present in the current object + storage. """ pass @abc.abstractmethod def add(self, content, obj_id=None, check_presence=True, *args, **kwargs): """Add a new object to the object storage. Args: content (bytes): object's raw content to add in storage. obj_id (bytes): checksum of [bytes] using [ID_HASH_ALGO] algorithm. When given, obj_id will be trusted to match the bytes. If missing, obj_id will be computed on the fly. check_presence (bool): indicate if the presence of the content should be verified before adding the file. Returns: the id (bytes) of the object into the storage. """ pass def add_batch(self, contents, check_presence=True): """Add a batch of new objects to the object storage. Args: contents (dict): mapping from obj_id to object conetnts Returns: the number of objects added to the storage """ ctr = 0 for obj_id, content in contents.items(): self.add(content, obj_id, check_presence=check_presence) ctr += 1 return ctr def restore(self, content, obj_id=None, *args, **kwargs): """Restore a content that have been corrupted. This function is identical to add but does not check if the object id is already in the file system. The default implementation provided by the current class is suitable for most cases. Args: content (bytes): object's raw content to add in storage obj_id (bytes): checksum of `bytes` as computed by ID_HASH_ALGO. When given, obj_id will be trusted to match bytes. If missing, obj_id will be computed on the fly. """ # check_presence to false will erase the potential previous content. return self.add(content, obj_id, check_presence=False) @abc.abstractmethod def get(self, obj_id, *args, **kwargs): """Retrieve the content of a given object. Args: obj_id (bytes): object id. Returns: the content of the requested object as bytes. Raises: ObjNotFoundError: if the requested object is missing. """ pass def get_batch(self, obj_ids, *args, **kwargs): """Retrieve objects' raw content in bulk from storage. Note: This function does have a default implementation in ObjStorage that is suitable for most cases. For object storages that needs to do the minimal number of requests possible (ex: remote object storages), that method - can be overriden to perform a more efficient operation. + can be overridden to perform a more efficient operation. Args: obj_ids ([bytes]: list of object ids. Returns: list of resulting contents, or None if the content could not be retrieved. Do not raise any exception as a fail for one content will not cancel the whole request. """ for obj_id in obj_ids: try: yield self.get(obj_id) except ObjNotFoundError: yield None @abc.abstractmethod def check(self, obj_id, *args, **kwargs): """Perform an integrity check for a given object. Verify that the file object is in place and that the gziped content matches the object id. Args: obj_id (bytes): object identifier. Raises: ObjNotFoundError: if the requested object is missing. Error: if the request object is corrupted. """ pass @abc.abstractmethod def delete(self, obj_id, *args, **kwargs): """Delete an object. Args: obj_id (bytes): object identifier. Raises: ObjNotFoundError: if the requested object is missing. """ if not self.allow_delete: raise PermissionError("Delete is not allowed.") # Management methods def get_random(self, batch_size, *args, **kwargs): """Get random ids of existing contents. This method is used in order to get random ids to perform content integrity verifications on random contents. Args: batch_size (int): Number of ids that will be given Yields: An iterable of ids (bytes) of contents that are in the current object storage. """ pass # Streaming methods def add_stream(self, content_iter, obj_id, check_presence=True): """Add a new object to the object storage using streaming. This function is identical to add() except it takes a generator that yields the chunked content instead of the whole content at once. Args: content (bytes): chunked generator that yields the object's raw content to add in storage. obj_id (bytes): object identifier check_presence (bool): indicate if the presence of the content should be verified before adding the file. Returns: the id (bytes) of the object into the storage. """ raise NotImplementedError def restore_stream(self, content_iter, obj_id=None): """Restore a content that have been corrupted using streaming. This function is identical to restore() except it takes a generator that yields the chunked content instead of the whole content at once. The default implementation provided by the current class is suitable for most cases. Args: content (bytes): chunked generator that yields the object's raw content to add in storage. obj_id (bytes): object identifier """ # check_presence to false will erase the potential previous content. return self.add_stream(content_iter, obj_id, check_presence=False) def get_stream(self, obj_id, chunk_size=DEFAULT_CHUNK_SIZE): """Retrieve the content of a given object as a chunked iterator. Args: obj_id (bytes): object id. Returns: the content of the requested object as bytes. Raises: ObjNotFoundError: if the requested object is missing. """ raise NotImplementedError diff --git a/swh/objstorage/objstorage_rados.py b/swh/objstorage/objstorage_rados.py index 82e41ca..9390287 100644 --- a/swh/objstorage/objstorage_rados.py +++ b/swh/objstorage/objstorage_rados.py @@ -1,87 +1,91 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import rados from swh.model import hashutil from swh.objstorage.exc import ObjNotFoundError from swh.objstorage import objstorage READ_SIZE = 8192 class RADOSObjStorage(objstorage.ObjStorage): """Object storage implemented with RADOS""" def __init__(self, *, rados_id, pool_name, ceph_config, allow_delete=False): super().__init__(allow_delete=allow_delete) self.pool_name = pool_name - self.cluster = rados.Rados(rados_id=rados_id, conf=ceph_config) + self.cluster = rados.Rados( + conf=ceph_config, + conffile='', + rados_id=rados_id, + ) self.cluster.connect() self.__ioctx = None def check_config(self, *, check_write): if self.pool_name not in self.cluster.list_pools(): raise ValueError('Pool %s does not exist' % self.pool_name) @staticmethod def _to_rados_obj_id(obj_id): """Convert to a RADOS object identifier""" return hashutil.hash_to_hex(obj_id) @property def ioctx(self): if not self.__ioctx: self.__ioctx = self.cluster.open_ioctx(self.pool_name) return self.__ioctx def __contains__(self, obj_id): try: self.ioctx.stat(self._to_rados_obj_id(obj_id)) except rados.ObjectNotFound: return False else: return True def add(self, content, obj_id=None, check_presence=True): if not obj_id: raise ValueError('add needs an obj_id') _obj_id = self._to_rados_obj_id(obj_id) if check_presence: try: self.ioctx.stat(_obj_id) except rados.ObjectNotFound: pass else: return obj_id self.ioctx.write_full(_obj_id, content) return obj_id def get(self, obj_id): chunks = [] _obj_id = self._to_rados_obj_id(obj_id) try: length, mtime = self.ioctx.stat(_obj_id) except rados.ObjectNotFound: raise ObjNotFoundError(obj_id) from None offset = 0 while offset < length: chunk = self.ioctx.read(_obj_id, offset, READ_SIZE) chunks.append(chunk) offset += len(chunk) return b''.join(chunks) def check(self, obj_id): return True def delete(self, obj_id): super().delete(obj_id) # check delete permission return True diff --git a/version.txt b/version.txt index 9fbb892..d194bfb 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.26-0-g5e9f3ca \ No newline at end of file +v0.0.27-0-g8bd0f8e \ No newline at end of file