diff --git a/swh/storage/vault/api/client.py b/swh/storage/vault/api/client.py --- a/swh/storage/vault/api/client.py +++ b/swh/storage/vault/api/client.py @@ -23,3 +23,13 @@ def directory_cook(self, obj_id): return self.post('vault/directory/%s/' % hashutil.hash_to_hex(obj_id), data={}) + + def revision_ls(self): + return self.get('vault/revision/') + + def revision_get(self, obj_id): + return self.get('vault/revision/%s/' % (hashutil.hash_to_hex(obj_id))) + + def revision_cook(self, obj_id): + return self.post('vault/revision/%s/' % hashutil.hash_to_hex(obj_id), + data={}) diff --git a/swh/storage/vault/api/cooking_tasks.py b/swh/storage/vault/api/cooking_tasks.py --- a/swh/storage/vault/api/cooking_tasks.py +++ b/swh/storage/vault/api/cooking_tasks.py @@ -6,12 +6,13 @@ from swh.scheduler.task import Task from swh.core import hashutil from ..cache import VaultCache -from ..cooker import DirectoryVaultCooker +from ..cooker import DirectoryVaultCooker, RevisionVaultCooker from ... import get_storage COOKER_TYPES = { - 'directory': DirectoryVaultCooker + 'directory': DirectoryVaultCooker, + 'revision': RevisionVaultCooker, } diff --git a/swh/storage/vault/cooker.py b/swh/storage/vault/cooker.py --- a/swh/storage/vault/cooker.py +++ b/swh/storage/vault/cooker.py @@ -11,6 +11,8 @@ import tarfile import tempfile +from pathlib import Path + from swh.core import hashutil @@ -21,6 +23,16 @@ HIDDEN_MESSAGE = (b'This content is hidden') +def get_tar_bytes(path, arcname=None): + path = Path(path) + if not arcname: + arcname = path.name + tar_buffer = io.BytesIO() + tar = tarfile.open(fileobj=tar_buffer, mode='w') + tar.add(str(path), arcname=arcname) + return tar_buffer.getbuffer() + + class BaseVaultCooker(metaclass=abc.ABCMeta): """Abstract base class for the vault's bundle creators @@ -110,6 +122,55 @@ pass +class RevisionVaultCooker(BaseVaultCooker): + """Cooker to create a directory bundle """ + CACHE_TYPE_KEY = 'revision' + + def __init__(self, storage, cache): + """Initialize a cooker that create revision bundles + + Args: + storage: source storage where content are retrieved. + cache: destination storage where the cooked bundle are stored. + + """ + self.storage = storage + self.cache = cache + + def cook(self, obj_id): + """Cook the requested revision into a Bundle + + Args: + obj_id (bytes): the id of the revision to be cooked. + + Returns: + bytes that correspond to the bundle + + """ + directory_cooker = DirectoryCooker(self.storage) + with tempfile.TemporaryDirectory(suffix='.cook') as root_tmp: + root = Path(root_tmp) + for revision in self.storage.revision_log([obj_id]): + revdir = root / hashutil.hash_to_hex(revision['id']) + revdir.mkdir() + directory_cooker.build_directory(revision['directory'], + str(revdir).encode()) + bundle_content = get_tar_bytes(root_tmp, + hashutil.hash_to_hex(obj_id)) + # Cache the bundle + self.update_cache(obj_id, bundle_content) + # Make a notification that the bundle have been cooked + # NOT YET IMPLEMENTED see TODO in function. + self.notify_bundle_ready( + notif_data='Bundle %s ready' % hashutil.hash_to_hex(obj_id), + bundle_id=obj_id) + + def notify_bundle_ready(self, notif_data, bundle_id): + # TODO plug this method with the notification method once + # done. + pass + + class DirectoryCooker(): """Creates a cooked directory from its sha1_git in the db. @@ -124,8 +185,18 @@ # Create temporary folder to retrieve the files into. root = bytes(tempfile.mkdtemp(prefix='directory.', suffix='.cook'), 'utf8') + self.build_directory(dir_id, root) + # Use the created directory to make a bundle with the data as + # a compressed directory. + bundle_content = self._create_bundle_content( + root, + hashutil.hash_to_hex(dir_id)) + return bundle_content + + def build_directory(self, dir_id, root): # Retrieve data from the database. data = self.storage.directory_ls(dir_id, recursive=True) + # Split into files and directory data. # TODO(seirl): also handle revision data. data1, data2 = itertools.tee(data, 2) @@ -136,13 +207,6 @@ self._create_tree(root, dir_data) self._create_files(root, file_data) - # Use the created directory to make a bundle with the data as - # a compressed directory. - bundle_content = self._create_bundle_content( - root, - hashutil.hash_to_hex(dir_id)) - return bundle_content - def _create_tree(self, root, directory_paths): """Create a directory tree from the given paths @@ -229,7 +293,4 @@ bytes that represent the compressed directory as a bundle. """ - tar_buffer = io.BytesIO() - tar = tarfile.open(fileobj=tar_buffer, mode='w') - tar.add(path.decode(), arcname=hex_dir_id) - return tar_buffer.getbuffer() + return get_tar_bytes(path.decode(), hex_dir_id)