diff --git a/PKG-INFO b/PKG-INFO index 35d6d2d..dc48822 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.objstorage -Version: 0.0.25 +Version: 0.0.26 Summary: Software Heritage Object Storage Home-page: https://forge.softwareheritage.org/diffusion/DOBJS Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/debian/control b/debian/control index c1eb8c3..60f2529 100644 --- a/debian/control +++ b/debian/control @@ -1,34 +1,41 @@ Source: swh-objstorage Maintainer: Software Heritage developers Section: python Priority: optional Build-Depends: debhelper (>= 9), dh-python (>= 2), python3-aiohttp (>= 2.1.0), python3-all, python3-flask, python3-nose, python3-setuptools, python3-swh.core (>= 0.0.37~), python3-swh.model (>= 0.0.14~), python3-click, python3-libcloud, python3-azure-storage, python3-vcversioner, python3-rados Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/diffusion/DOBJS/ Package: python3-swh.objstorage Architecture: all Depends: python3-swh.core (>= 0.0.37~), ${misc:Depends}, ${python3:Depends}, python3-aiohttp (>= 2.1.0) Breaks: python3-swh.archiver (<< 0.0.3~) Description: Software Heritage Object Storage Package: python3-swh.objstorage.cloud Architecture: all Depends: python3-swh.objstorage (= ${binary:Version}), python3-libcloud, python3-azure-storage, ${misc:Depends}, ${python3:Depends} Breaks: python3-swh.objstorage (<= 0.0.7~) Description: Software Heritage Cloud Object Storage +Package: python3-swh.objstorage.rados +Architecture: all +Depends: python3-swh.objstorage (= ${binary:Version}), python3-rados + ${misc:Depends}, ${python3:Depends} +Breaks: python3-swh.objstorage (<= 0.0.25~) +Description: Software Heritage Object Storage based on RADOS + diff --git a/debian/rules b/debian/rules index 7f4935e..9487559 100755 --- a/debian/rules +++ b/debian/rules @@ -1,19 +1,22 @@ #!/usr/bin/make -f export PYBUILD_NAME=swh.objstorage export PYBUILD_TEST_ARGS=--with-doctest -sv -a !db,!fs %: dh $@ --with python3 --buildsystem=pybuild override_dh_install: dh_install rm -v $(CURDIR)/debian/python3-*/usr/lib/python*/dist-packages/swh/__init__.py for pyvers in $(shell py3versions -vr); do \ mkdir -p $(CURDIR)/debian/python3-swh.objstorage.cloud/usr/lib/python$$pyvers/dist-packages/swh/objstorage/cloud ; \ + mkdir -p $(CURDIR)/debian/python3-swh.objstorage.rados/usr/lib/python$$pyvers/dist-packages/swh/objstorage ; \ mv $(CURDIR)/debian/python3-swh.objstorage/usr/lib/python$$pyvers/dist-packages/swh/objstorage/cloud/* \ $(CURDIR)/debian/python3-swh.objstorage.cloud/usr/lib/python$$pyvers/dist-packages/swh/objstorage/cloud/ ; \ + mv $(CURDIR)/debian/python3-swh.objstorage/usr/lib/python$$pyvers/dist-packages/swh/objstorage/objstorage_rados.py \ + $(CURDIR)/debian/python3-swh.objstorage.rados/usr/lib/python$$pyvers/dist-packages/swh/objstorage/ ; \ done override_dh_auto_test: no_proxy=127.0.0.1 dh_auto_test diff --git a/swh.objstorage.egg-info/PKG-INFO b/swh.objstorage.egg-info/PKG-INFO index 35d6d2d..dc48822 100644 --- a/swh.objstorage.egg-info/PKG-INFO +++ b/swh.objstorage.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.objstorage -Version: 0.0.25 +Version: 0.0.26 Summary: Software Heritage Object Storage Home-page: https://forge.softwareheritage.org/diffusion/DOBJS Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh/objstorage/multiplexer/striping_objstorage.py b/swh/objstorage/multiplexer/striping_objstorage.py index b63222d..4e756ef 100644 --- a/swh/objstorage/multiplexer/striping_objstorage.py +++ b/swh/objstorage/multiplexer/striping_objstorage.py @@ -1,71 +1,71 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from collections import defaultdict import queue from .multiplexer_objstorage import ObjStorageThread, MultiplexerObjStorage class StripingObjStorage(MultiplexerObjStorage): """Stripes objects across multiple objstorages This objstorage implementation will write objects to objstorages in a predictable way: it takes the modulo of the last 8 bytes of the object identifier with the number of object storages passed, which will yield an (almost) even distribution. Objects are read from all storages in turn until it succeeds. """ MOD_BYTES = 8 def __init__(self, storages, **kwargs): super().__init__(storages, **kwargs) self.num_storages = len(storages) def get_storage_index(self, obj_id): if obj_id is None: raise ValueError( 'StripingObjStorage always needs obj_id to be set' ) - index = int.from_bytes(obj_id[:-self.MOD_BYTES], 'little') + index = int.from_bytes(obj_id[:-self.MOD_BYTES], 'big') return index % self.num_storages def get_write_threads(self, obj_id): idx = self.get_storage_index(obj_id) yield self.storage_threads[idx] def get_read_threads(self, obj_id=None): if obj_id: idx = self.get_storage_index(obj_id) else: idx = 0 for i in range(self.num_storages): yield self.storage_threads[(idx + i) % self.num_storages] def add_batch(self, contents, check_presence=True): """Add a batch of new objects to the object storage. """ content_by_storage_index = defaultdict(dict) for obj_id, content in contents.items(): storage_index = self.get_storage_index(obj_id) content_by_storage_index[storage_index][obj_id] = content mailbox = queue.Queue() for storage_index, contents in content_by_storage_index.items(): self.storage_threads[storage_index].queue_command( 'add_batch', contents, check_presence=check_presence, mailbox=mailbox, ) return sum( ObjStorageThread.collect_results( mailbox, len(content_by_storage_index) ) ) diff --git a/version.txt b/version.txt index 2633902..9fbb892 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.25-0-g418dfe7 \ No newline at end of file +v0.0.26-0-g5e9f3ca \ No newline at end of file