diff --git a/swh/storage/vault/api/client.py b/swh/storage/vault/api/client.py --- a/swh/storage/vault/api/client.py +++ b/swh/storage/vault/api/client.py @@ -14,22 +14,14 @@ def __init__(self, base_url): super().__init__(api_exception=StorageAPIError, url=base_url) - def directory_ls(self): - return self.get('vault/directory/') + def ls(self, obj_type): + return self.get('vault/{}/'.format(obj_type)) - def directory_get(self, obj_id): - return self.get('vault/directory/%s/' % (hashutil.hash_to_hex(obj_id))) + def get(self, obj_type, obj_id): + return self.get('vault/{}/{}/'.format(obj_type, + hashutil.hash_to_hex(obj_id))) - def directory_cook(self, obj_id): - return self.post('vault/directory/%s/' % hashutil.hash_to_hex(obj_id), - data={}) - - def revision_ls(self): - return self.get('vault/revision/') - - def revision_get(self, obj_id): - return self.get('vault/revision/%s/' % (hashutil.hash_to_hex(obj_id))) - - def revision_cook(self, obj_id): - return self.post('vault/revision/%s/' % hashutil.hash_to_hex(obj_id), + def cook(self, obj_type, obj_id): + return self.post('vault/{}/{}/'.format(obj_type, + hashutil.hash_to_hex(obj_id)), data={}) diff --git a/swh/storage/vault/api/cooking_tasks.py b/swh/storage/vault/api/cooking_tasks.py --- a/swh/storage/vault/api/cooking_tasks.py +++ b/swh/storage/vault/api/cooking_tasks.py @@ -6,16 +6,10 @@ from swh.scheduler.task import Task from swh.core import hashutil from ..cache import VaultCache -from ..cooker import DirectoryVaultCooker, RevisionVaultCooker +from ..cooker import COOKER_TYPES from ... import get_storage -COOKER_TYPES = { - 'directory': DirectoryVaultCooker, - 'revision': RevisionVaultCooker, -} - - class SWHCookingTask(Task): """Main task which archives a contents batch. diff --git a/swh/storage/vault/api/server.py b/swh/storage/vault/api/server.py --- a/swh/storage/vault/api/server.py +++ b/swh/storage/vault/api/server.py @@ -4,16 +4,16 @@ # See top-level LICENSE file for more information import click +import re from flask import abort, g from werkzeug.routing import BaseConverter from swh.core import config from swh.core.api import (SWHServerAPIApp, error_handler, encode_data_server as encode_data) -from swh.storage import get_storage -from swh.storage.vault.api import cooking_tasks # NOQA +from swh.scheduler.utils import get_task +from swh.storage.vault.api.cooking_tasks import COOKER_TYPES +from swh.storage.vault.api.cooking_tasks import SWHCookingTask # noqa from swh.storage.vault.cache import VaultCache -from swh.storage.vault.cooker import DirectoryVaultCooker -from swh.scheduler.celery_backend.config import app as celery_app cooking_task_name = 'swh.storage.vault.api.cooking_tasks.SWHCookingTask' @@ -34,14 +34,15 @@ } -class RegexConverter(BaseConverter): +class CookerConverter(BaseConverter): def __init__(self, url_map, *items): super().__init__(url_map) - self.regex = items[0] + types = [re.escape(c) for c in COOKER_TYPES] + self.regex = '({})'.format('|'.join(types)) app = SWHServerAPIApp(__name__) -app.url_map.converters['regex'] = RegexConverter +app.url_map.converters['cooker'] = CookerConverter @app.errorhandler(Exception) @@ -52,10 +53,6 @@ @app.before_request def before_request(): g.cache = VaultCache(**app.config['cache']) - g.cooker = DirectoryVaultCooker( - get_storage(**app.config['storage']), - g.cache - ) @app.route('/') @@ -63,7 +60,7 @@ return 'SWH vault API server' -@app.route('/vault//', +@app.route('/vault//', methods=['GET']) def ls_directory(type): return encode_data(list( @@ -71,7 +68,7 @@ )) -@app.route('/vault///', +@app.route('/vault///', methods=['GET']) def get_cooked_directory(type, id): if not g.cache.is_cached(type, id): @@ -79,10 +76,10 @@ return encode_data(g.cache.get(type, id).decode()) -@app.route('/vault///', +@app.route('/vault///', methods=['POST']) def cook_request_directory(type, id): - task = celery_app.tasks[cooking_task_name] + task = get_task(cooking_task_name) task.delay(type, id, app.config['storage'], app.config['cache']) # Return url to get the content and 201 CREATED return encode_data('/vault/%s/%s/' % (type, id)), 201 diff --git a/swh/storage/vault/cache.py b/swh/storage/vault/cache.py --- a/swh/storage/vault/cache.py +++ b/swh/storage/vault/cache.py @@ -9,12 +9,6 @@ from swh.objstorage import get_objstorage from swh.objstorage.objstorage_pathslicing import DIR_MODE -BUNDLE_TYPES = { - 'directory': 'd', - 'revision': 'r', - 'snapshot': 's', -} - class VaultCache(): """The vault cache is an object storage that stores bundles @@ -26,24 +20,8 @@ """ def __init__(self, root): - for subdir in BUNDLE_TYPES.values(): - fp = os.path.join(root, subdir) - if not os.path.isdir(fp): - os.makedirs(fp, DIR_MODE, exist_ok=True) - - self.storages = { - type: get_objstorage( - 'pathslicing', {'root': os.path.join(root, subdir), - 'slicing': '0:1/0:5'} - ) - for type, subdir in BUNDLE_TYPES.items() - } - - def __contains__(self, obj_id): - for storage in self.storages: - if obj_id in storage: - return True - return False + self.root = root + self.storages = {} def add(self, obj_type, obj_id, content): storage = self._get_storage(obj_type) @@ -63,7 +41,13 @@ def _get_storage(self, obj_type): """Get the storage that corresponds to the object type""" - try: - return self.storages[obj_type] - except: - raise ValueError('Wrong bundle type: ' + obj_type) + + if obj_type not in self.storages: + fp = os.path.join(self.root, obj_type) + if not os.path.isdir(fp): + os.makedirs(fp, DIR_MODE, exist_ok=True) + + self.storages[obj_type] = get_objstorage( + 'pathslicing', {'root': fp, 'slicing': '0:1/0:5'}) + + return self.storages[obj_type] diff --git a/swh/storage/vault/cooker.py b/swh/storage/vault/cooker.py --- a/swh/storage/vault/cooker.py +++ b/swh/storage/vault/cooker.py @@ -79,21 +79,10 @@ pass -class DirectoryVaultCooker(BaseVaultCooker): +class DirectoryCooker(BaseVaultCooker): """Cooker to create a directory bundle """ CACHE_TYPE_KEY = 'directory' - def __init__(self, storage, cache): - """Initialize a cooker that create directory bundles - - Args: - storage: source storage where content are retrieved. - cache: destination storage where the cooked bundle are stored. - - """ - self.storage = storage - self.cache = cache - def cook(self, obj_id): """Cook the requested directory into a Bundle @@ -106,7 +95,7 @@ """ # Create the bytes that corresponds to the compressed # directory. - directory_cooker = DirectoryCooker(self.storage) + directory_cooker = DirectoryBuilder(self.storage) bundle_content = directory_cooker.get_directory_bytes(obj_id) # Cache the bundle self.update_cache(obj_id, bundle_content) @@ -122,20 +111,9 @@ pass -class RevisionVaultCooker(BaseVaultCooker): +class RevisionFlatCooker(BaseVaultCooker): """Cooker to create a directory bundle """ - CACHE_TYPE_KEY = 'revision' - - def __init__(self, storage, cache): - """Initialize a cooker that create revision bundles - - Args: - storage: source storage where content are retrieved. - cache: destination storage where the cooked bundle are stored. - - """ - self.storage = storage - self.cache = cache + CACHE_TYPE_KEY = 'revision_flat' def cook(self, obj_id): """Cook the requested revision into a Bundle @@ -147,7 +125,7 @@ bytes that correspond to the bundle """ - directory_cooker = DirectoryCooker(self.storage) + directory_cooker = DirectoryBuilder(self.storage) with tempfile.TemporaryDirectory(suffix='.cook') as root_tmp: root = Path(root_tmp) for revision in self.storage.revision_log([obj_id]): @@ -171,7 +149,7 @@ pass -class DirectoryCooker(): +class DirectoryBuilder: """Creates a cooked directory from its sha1_git in the db. Warning: This is NOT a directly accessible cooker, but a low-level @@ -294,3 +272,9 @@ """ return get_tar_bytes(path.decode(), hex_dir_id) + + +COOKER_TYPES = { + 'directory': DirectoryCooker, + 'revision_flat': RevisionFlatCooker, +}