diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,3 +1,3 @@ swh.core[http] >= 0.3 swh.model >= 0.0.27 -swh.perfecthash +swh.perfecthash >= 0.1.2 diff --git a/swh/objstorage/backends/winery/objstorage.py b/swh/objstorage/backends/winery/objstorage.py --- a/swh/objstorage/backends/winery/objstorage.py +++ b/swh/objstorage/backends/winery/objstorage.py @@ -6,8 +6,9 @@ import logging from multiprocessing import Process +from swh.model import hashutil from swh.objstorage import exc -from swh.objstorage.objstorage import ObjStorage, compute_hash +from swh.objstorage.objstorage import ObjStorage from .roshard import ROShard from .rwshard import RWShard @@ -17,6 +18,11 @@ logger = logging.getLogger(__name__) +def compute_hash(content): + algo = "sha256" + return hashutil.MultiHash.from_data(content, hash_names=[algo],).digest().get(algo) + + class WineryObjStorage(ObjStorage): def __init__(self, **kwargs): super().__init__(**kwargs) @@ -63,10 +69,16 @@ class WineryReader(WineryBase): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.shards = {} + def roshard(self, name): - shard = ROShard(name, **self.args) - shard.load() - return shard + if name not in self.shards: + shard = ROShard(name, **self.args) + shard.load() + self.shards[name] = shard + return self.shards[name] def get(self, obj_id): shard_info = self.base.get(obj_id) diff --git a/swh/objstorage/tests/test_objstorage_winery.py b/swh/objstorage/tests/test_objstorage_winery.py --- a/swh/objstorage/tests/test_objstorage_winery.py +++ b/swh/objstorage/tests/test_objstorage_winery.py @@ -96,7 +96,10 @@ shard = winery.base.whoami content = b"SOMETHING" obj_id = winery.add(content=content) - assert obj_id.hex() == "0c8c841f7d9fd4874d841506d3ffc16808b1d579" + assert ( + obj_id.hex() + == "866878b165607851782d8d233edf0c261172ff67926330d3bbd10c705b92d24f" + ) assert winery.add(content=content, obj_id=obj_id) == obj_id assert winery.add(content=content, obj_id=obj_id, check_presence=False) == obj_id assert winery.base.whoami == shard @@ -112,7 +115,10 @@ shard = winery.base.whoami content = b"SOMETHING" obj_id = winery.add(content=content) - assert obj_id.hex() == "0c8c841f7d9fd4874d841506d3ffc16808b1d579" + assert ( + obj_id.hex() + == "866878b165607851782d8d233edf0c261172ff67926330d3bbd10c705b92d24f" + ) assert winery.base.whoami != shard assert len(winery.packers) == 1 packer = winery.packers[0] diff --git a/winery-test-environment/grid5000.yml b/winery-test-environment/grid5000.yml --- a/winery-test-environment/grid5000.yml +++ b/winery-test-environment/grid5000.yml @@ -39,11 +39,14 @@ tasks: # do that before lvm gets a chance to investigate and get the wrong idea - # about /dev/sdc on grid5000 because there surely will be leftovers from + # about /dev/disk2 on grid5000 because there surely will be leftovers from # whoever used the machine last - - name: zap /dev/sdc + - name: clear leftovers from the disk to be used for OSDs shell: | - dd if=/dev/zero of=/dev/sdc count=100 bs=1024k + dd if=/dev/zero of=/dev/disk2 count=100 bs=1024k + touch /etc/dd.done + args: + creates: /etc/dd.done - hosts: all become: true diff --git a/winery-test-environment/osd.yml b/winery-test-environment/osd.yml --- a/winery-test-environment/osd.yml +++ b/winery-test-environment/osd.yml @@ -33,8 +33,8 @@ # the desired side effect here is twofold # * device zap blocks until the osd daemon is ready on the target host - # * on grid5000 /dev/sdc needs to be applied - - name: zap /dev/sdc + # * on grid5000 /dev/disk2 needs to be applied + - name: zap /dev/disk2 shell: | - ceph orch device zap {{ inventory_hostname }} /dev/sdc --force || true + ceph orch device zap {{ inventory_hostname }} /dev/disk2 --force || true delegate_to: ceph1