Changeset View
Changeset View
Standalone View
Standalone View
swh/vault/cookers/base.py
# Copyright (C) 2016-2018 The Software Heritage developers | # Copyright (C) 2016-2018 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import abc | import abc | ||||
import io | import io | ||||
import logging | import logging | ||||
from swh.core import config | from swh.core import config | ||||
from swh.model import hashutil | from swh.model import hashutil | ||||
from swh.storage import get_storage | from swh.storage import get_storage | ||||
from swh.vault.api.client import RemoteVaultClient | from swh.vault import get_vault | ||||
DEFAULT_CONFIG_PATH = 'vault/cooker' | DEFAULT_CONFIG_PATH = 'vault/cooker' | ||||
DEFAULT_CONFIG = { | DEFAULT_CONFIG = { | ||||
'storage': ('dict', { | 'storage': ('dict', { | ||||
'cls': 'remote', | 'cls': 'remote', | ||||
'args': { | 'args': { | ||||
'url': 'http://localhost:5002/', | 'url': 'http://localhost:5002/', | ||||
}, | }, | ||||
}), | }), | ||||
'vault_url': ('str', 'http://localhost:5005/'), | 'vault': ('dict', { | ||||
'cls': 'remote', | |||||
'args': { | |||||
'url': 'http://localhost:5005/', | |||||
}, | |||||
}), | |||||
'max_bundle_size': ('int', 2 ** 29), # 512 MiB | 'max_bundle_size': ('int', 2 ** 29), # 512 MiB | ||||
} | } | ||||
class PolicyError(Exception): | class PolicyError(Exception): | ||||
"""Raised when the bundle violates the cooking policy.""" | """Raised when the bundle violates the cooking policy.""" | ||||
pass | pass | ||||
Show All 23 Lines | class BaseVaultCooker(metaclass=abc.ABCMeta): | ||||
This class describes a common API for the cookers. | This class describes a common API for the cookers. | ||||
To define a new cooker, inherit from this class and override: | To define a new cooker, inherit from this class and override: | ||||
- CACHE_TYPE_KEY: key to use for the bundle to reference in cache | - CACHE_TYPE_KEY: key to use for the bundle to reference in cache | ||||
- def cook(): cook the object into a bundle | - def cook(): cook the object into a bundle | ||||
""" | """ | ||||
CACHE_TYPE_KEY = None | CACHE_TYPE_KEY = None | ||||
def __init__(self, obj_type, obj_id, *, override_cfg=None): | def __init__(self, obj_type, obj_id, backend=None, storage=None, | ||||
override_cfg=None): | |||||
"""Initialize the cooker. | """Initialize the cooker. | ||||
The type of the object represented by the id depends on the | The type of the object represented by the id depends on the | ||||
concrete class. Very likely, each type of bundle will have its | concrete class. Very likely, each type of bundle will have its | ||||
own cooker class. | own cooker class. | ||||
Args: | Args: | ||||
storage: the storage object | storage: the storage object | ||||
cache: the cache where to store the bundle | cache: the cache where to store the bundle | ||||
obj_id: id of the object to be cooked into a bundle. | obj_id: id of the object to be cooked into a bundle. | ||||
""" | """ | ||||
self.config = config.load_named_config(DEFAULT_CONFIG_PATH, | self.config = config.load_named_config(DEFAULT_CONFIG_PATH, | ||||
DEFAULT_CONFIG) | DEFAULT_CONFIG) | ||||
if override_cfg is not None: | if override_cfg is not None: | ||||
self.config.update(override_cfg) | self.config.update(override_cfg) | ||||
self.obj_type = obj_type | self.obj_type = obj_type | ||||
self.obj_id = hashutil.hash_to_bytes(obj_id) | self.obj_id = hashutil.hash_to_bytes(obj_id) | ||||
self.backend = RemoteVaultClient(self.config['vault_url']) | if backend is None: | ||||
self.storage = get_storage(**self.config['storage']) | backend = get_vault(**self.config['vault']) | ||||
self.backend = backend | |||||
if storage is None: | |||||
storage = get_storage(**self.config['storage']) | |||||
self.storage = storage | |||||
self.max_bundle_size = self.config['max_bundle_size'] | self.max_bundle_size = self.config['max_bundle_size'] | ||||
@abc.abstractmethod | @abc.abstractmethod | ||||
def check_exists(self): | def check_exists(self): | ||||
"""Checks that the requested object exists and can be cooked. | """Checks that the requested object exists and can be cooked. | ||||
Override this in the cooker implementation. | Override this in the cooker implementation. | ||||
""" | """ | ||||
Show All 39 Lines |