diff --git a/swh/storage/vault/api/client.py b/swh/storage/vault/api/client.py --- a/swh/storage/vault/api/client.py +++ b/swh/storage/vault/api/client.py @@ -17,7 +17,7 @@ def ls(self, obj_type): return self.get('vault/{}/'.format(obj_type)) - def get(self, obj_type, obj_id): + def fetch(self, obj_type, obj_id): return self.get('vault/{}/{}/'.format(obj_type, hashutil.hash_to_hex(obj_id))) diff --git a/swh/storage/vault/api/cooking_tasks.py b/swh/storage/vault/api/cooking_tasks.py --- a/swh/storage/vault/api/cooking_tasks.py +++ b/swh/storage/vault/api/cooking_tasks.py @@ -6,7 +6,7 @@ from swh.scheduler.task import Task from swh.core import hashutil from ..cache import VaultCache -from ..cooker import COOKER_TYPES +from ..cookers import COOKER_TYPES from ... import get_storage diff --git a/swh/storage/vault/api/server.py b/swh/storage/vault/api/server.py --- a/swh/storage/vault/api/server.py +++ b/swh/storage/vault/api/server.py @@ -11,9 +11,9 @@ from swh.core.api import (SWHServerAPIApp, error_handler, encode_data_server as encode_data) from swh.scheduler.utils import get_task -from swh.storage.vault.api.cooking_tasks import COOKER_TYPES from swh.storage.vault.api.cooking_tasks import SWHCookingTask # noqa from swh.storage.vault.cache import VaultCache +from swh.storage.vault.cookers import COOKER_TYPES cooking_task_name = 'swh.storage.vault.api.cooking_tasks.SWHCookingTask' diff --git a/swh/storage/vault/cookers/__init__.py b/swh/storage/vault/cookers/__init__.py new file mode 100644 --- /dev/null +++ b/swh/storage/vault/cookers/__init__.py @@ -0,0 +1,12 @@ +# Copyright (C) 2016 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from .directory import DirectoryCooker +from .revision_flat import RevisionFlatCooker + +COOKER_TYPES = { + 'directory': DirectoryCooker, + 'revision_flat': RevisionFlatCooker, +} diff --git a/swh/storage/vault/cooker.py b/swh/storage/vault/cookers/base.py rename from swh/storage/vault/cooker.py rename to swh/storage/vault/cookers/base.py --- a/swh/storage/vault/cooker.py +++ b/swh/storage/vault/cookers/base.py @@ -16,13 +16,6 @@ from swh.core import hashutil -SKIPPED_MESSAGE = (b'This content have not been retrieved in ' - b'Software Heritage archive due to its size') - - -HIDDEN_MESSAGE = (b'This content is hidden') - - def get_tar_bytes(path, arcname=None): path = Path(path) if not arcname: @@ -33,6 +26,13 @@ return tar_buffer.getbuffer() +SKIPPED_MESSAGE = (b'This content have not been retrieved in ' + b'Software Heritage archive due to its size') + + +HIDDEN_MESSAGE = (b'This content is hidden') + + class BaseVaultCooker(metaclass=abc.ABCMeta): """Abstract base class for the vault's bundle creators @@ -79,76 +79,6 @@ pass -class DirectoryCooker(BaseVaultCooker): - """Cooker to create a directory bundle """ - CACHE_TYPE_KEY = 'directory' - - def cook(self, obj_id): - """Cook the requested directory into a Bundle - - Args: - obj_id (bytes): the id of the directory to be cooked. - - Returns: - bytes that correspond to the bundle - - """ - # Create the bytes that corresponds to the compressed - # directory. - directory_cooker = DirectoryBuilder(self.storage) - bundle_content = directory_cooker.get_directory_bytes(obj_id) - # Cache the bundle - self.update_cache(obj_id, bundle_content) - # Make a notification that the bundle have been cooked - # NOT YET IMPLEMENTED see TODO in function. - self.notify_bundle_ready( - notif_data='Bundle %s ready' % hashutil.hash_to_hex(obj_id), - bundle_id=obj_id) - - def notify_bundle_ready(self, notif_data, bundle_id): - # TODO plug this method with the notification method once - # done. - pass - - -class RevisionFlatCooker(BaseVaultCooker): - """Cooker to create a directory bundle """ - CACHE_TYPE_KEY = 'revision_flat' - - def cook(self, obj_id): - """Cook the requested revision into a Bundle - - Args: - obj_id (bytes): the id of the revision to be cooked. - - Returns: - bytes that correspond to the bundle - - """ - directory_cooker = DirectoryBuilder(self.storage) - with tempfile.TemporaryDirectory(suffix='.cook') as root_tmp: - root = Path(root_tmp) - for revision in self.storage.revision_log([obj_id]): - revdir = root / hashutil.hash_to_hex(revision['id']) - revdir.mkdir() - directory_cooker.build_directory(revision['directory'], - str(revdir).encode()) - bundle_content = get_tar_bytes(root_tmp, - hashutil.hash_to_hex(obj_id)) - # Cache the bundle - self.update_cache(obj_id, bundle_content) - # Make a notification that the bundle have been cooked - # NOT YET IMPLEMENTED see TODO in function. - self.notify_bundle_ready( - notif_data='Bundle %s ready' % hashutil.hash_to_hex(obj_id), - bundle_id=obj_id) - - def notify_bundle_ready(self, notif_data, bundle_id): - # TODO plug this method with the notification method once - # done. - pass - - class DirectoryBuilder: """Creates a cooked directory from its sha1_git in the db. @@ -272,9 +202,3 @@ """ return get_tar_bytes(path.decode(), hex_dir_id) - - -COOKER_TYPES = { - 'directory': DirectoryCooker, - 'revision_flat': RevisionFlatCooker, -} diff --git a/swh/storage/vault/cookers/directory.py b/swh/storage/vault/cookers/directory.py new file mode 100644 --- /dev/null +++ b/swh/storage/vault/cookers/directory.py @@ -0,0 +1,39 @@ +# Copyright (C) 2016 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from .base import BaseVaultCooker, DirectoryBuilder +from swh.core import hashutil + + +class DirectoryCooker(BaseVaultCooker): + """Cooker to create a directory bundle """ + CACHE_TYPE_KEY = 'directory' + + def cook(self, obj_id): + """Cook the requested directory into a Bundle + + Args: + obj_id (bytes): the id of the directory to be cooked. + + Returns: + bytes that correspond to the bundle + + """ + # Create the bytes that corresponds to the compressed + # directory. + directory_cooker = DirectoryBuilder(self.storage) + bundle_content = directory_cooker.get_directory_bytes(obj_id) + # Cache the bundle + self.update_cache(obj_id, bundle_content) + # Make a notification that the bundle have been cooked + # NOT YET IMPLEMENTED see TODO in function. + self.notify_bundle_ready( + notif_data='Bundle %s ready' % hashutil.hash_to_hex(obj_id), + bundle_id=obj_id) + + def notify_bundle_ready(self, notif_data, bundle_id): + # TODO plug this method with the notification method once + # done. + pass diff --git a/swh/storage/vault/cookers/revision_flat.py b/swh/storage/vault/cookers/revision_flat.py new file mode 100644 --- /dev/null +++ b/swh/storage/vault/cookers/revision_flat.py @@ -0,0 +1,49 @@ +# Copyright (C) 2016 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import tempfile +from pathlib import Path + +from swh.core import hashutil + +from .base import BaseVaultCooker, DirectoryBuilder, get_tar_bytes + + +class RevisionFlatCooker(BaseVaultCooker): + """Cooker to create a directory bundle """ + CACHE_TYPE_KEY = 'revision_flat' + + def cook(self, obj_id): + """Cook the requested revision into a Bundle + + Args: + obj_id (bytes): the id of the revision to be cooked. + + Returns: + bytes that correspond to the bundle + + """ + directory_cooker = DirectoryBuilder(self.storage) + with tempfile.TemporaryDirectory(suffix='.cook') as root_tmp: + root = Path(root_tmp) + for revision in self.storage.revision_log([obj_id]): + revdir = root / hashutil.hash_to_hex(revision['id']) + revdir.mkdir() + directory_cooker.build_directory(revision['directory'], + str(revdir).encode()) + bundle_content = get_tar_bytes(root_tmp, + hashutil.hash_to_hex(obj_id)) + # Cache the bundle + self.update_cache(obj_id, bundle_content) + # Make a notification that the bundle have been cooked + # NOT YET IMPLEMENTED see TODO in function. + self.notify_bundle_ready( + notif_data='Bundle %s ready' % hashutil.hash_to_hex(obj_id), + bundle_id=obj_id) + + def notify_bundle_ready(self, notif_data, bundle_id): + # TODO plug this method with the notification method once + # done. + pass