Changeset View
Changeset View
Standalone View
Standalone View
swh/objstorage/backends/winery/objstorage.py
| # Copyright (C) 2022 The Software Heritage developers | # Copyright (C) 2022 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import logging | import logging | ||||
| from multiprocessing import Process | from multiprocessing import Process | ||||
| from swh.objstorage import exc | from swh.objstorage import exc | ||||
| from swh.objstorage.objstorage import ObjStorage, compute_hash | from swh.objstorage.objstorage import ObjStorage, compute_hash | ||||
| from .roshard import ROShard | from .roshard import ROShard | ||||
| from .rwshard import RWShard | from .rwshard import RWShard | ||||
| from .sharedbase import SharedBase | from .sharedbase import SharedBase | ||||
| from .stats import Stats | |||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| class WineryObjStorage(ObjStorage): | class WineryObjStorage(ObjStorage): | ||||
| def __init__(self, **kwargs): | def __init__(self, **kwargs): | ||||
| super().__init__(**kwargs) | super().__init__(**kwargs) | ||||
| if kwargs.get("readonly"): | if kwargs.get("readonly"): | ||||
| ▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines | def get(self, obj_id): | ||||
| raise exc.ObjNotFoundError(obj_id) | raise exc.ObjNotFoundError(obj_id) | ||||
| return content | return content | ||||
| def pack(shard, **kwargs): | def pack(shard, **kwargs): | ||||
| return Packer(shard, **kwargs).run() | return Packer(shard, **kwargs).run() | ||||
| class Packer: | class Packer(Stats): | ||||
| def __init__(self, shard, **kwargs): | def __init__(self, shard, **kwargs): | ||||
| super().__init__(kwargs.get("output_dir")) | |||||
| self.args = kwargs | self.args = kwargs | ||||
| self.shard = shard | self.shard = shard | ||||
| self.init() | self.init() | ||||
| def init(self): | def init(self): | ||||
| self.rw = RWShard(self.shard, **self.args) | self.rw = RWShard(self.shard, **self.args) | ||||
| self.ro = ROShard(self.shard, **self.args) | self.ro = ROShard(self.shard, **self.args) | ||||
| def uninit(self): | def uninit(self): | ||||
| del self.ro | del self.ro | ||||
| self.rw.uninit() | self.rw.uninit() | ||||
| def run(self): | def run(self): | ||||
| self.ro.create(self.rw.count()) | self.ro.create(self.rw.count()) | ||||
| for obj_id, content in self.rw.all(): | for obj_id, content in self.rw.all(): | ||||
| self.ro.add(content, obj_id) | self.ro.add(content, obj_id) | ||||
| if self.stats_active: | |||||
| self.stats_read(obj_id, content) | |||||
| self.stats_write(obj_id, content) | |||||
| self.ro.save() | self.ro.save() | ||||
| base = SharedBase(**self.args) | base = SharedBase(**self.args) | ||||
| base.shard_packing_ends(self.shard) | base.shard_packing_ends(self.shard) | ||||
| base.uninit() | base.uninit() | ||||
| self.rw.uninit() | self.rw.uninit() | ||||
| self.rw.drop() | self.rw.drop() | ||||
| return True | return True | ||||
| ▲ Show 20 Lines • Show All 52 Lines • Show Last 20 Lines | |||||