Page MenuHomeSoftware Heritage
Files F9345645

D178.diff
No OneTemporary
Actions

D178.diff
View Options

diff --git a/swh/storage/vault/api/client.py b/swh/storage/vault/api/client.py
--- a/swh/storage/vault/api/client.py
+++ b/swh/storage/vault/api/client.py
@@ -23,3 +23,13 @@
def directory_cook(self, obj_id):
return self.post('vault/directory/%s/' % hashutil.hash_to_hex(obj_id),
data={})
+
+ def revision_ls(self):
+ return self.get('vault/revision/')
+
+ def revision_get(self, obj_id):
+ return self.get('vault/revision/%s/' % (hashutil.hash_to_hex(obj_id)))
+
+ def revision_cook(self, obj_id):
+ return self.post('vault/revision/%s/' % hashutil.hash_to_hex(obj_id),
+ data={})
diff --git a/swh/storage/vault/api/cooking_tasks.py b/swh/storage/vault/api/cooking_tasks.py
--- a/swh/storage/vault/api/cooking_tasks.py
+++ b/swh/storage/vault/api/cooking_tasks.py
@@ -6,12 +6,13 @@
from swh.scheduler.task import Task
from swh.core import hashutil
from ..cache import VaultCache
-from ..cooker import DirectoryVaultCooker
+from ..cooker import DirectoryVaultCooker, RevisionVaultCooker
from ... import get_storage
COOKER_TYPES = {
- 'directory': DirectoryVaultCooker
+ 'directory': DirectoryVaultCooker,
+ 'revision': RevisionVaultCooker,
}
diff --git a/swh/storage/vault/cooker.py b/swh/storage/vault/cooker.py
--- a/swh/storage/vault/cooker.py
+++ b/swh/storage/vault/cooker.py
@@ -11,6 +11,8 @@
import tarfile
import tempfile
+from pathlib import Path
+
from swh.core import hashutil
@@ -110,6 +112,60 @@
pass
+class RevisionVaultCooker(BaseVaultCooker):
+ """Cooker to create a directory bundle """
+ CACHE_TYPE_KEY = 'revision'
+
+ def __init__(self, storage, cache):
+ """Initialize a cooker that create revision bundles
+
+ Args:
+ storage: source storage where content are retrieved.
+ cache: destination storage where the cooked bundle are stored.
+
+ """
+ self.storage = storage
+ self.cache = cache
+
+ def cook(self, obj_id):
+ """Cook the requested revision into a Bundle
+
+ Args:
+ obj_id (bytes): the id of the revision to be cooked.
+
+ Returns:
+ bytes that correspond to the bundle
+
+ """
+ directory_cooker = DirectoryCooker(self.storage)
+ with tempfile.TemporaryDirectory(suffix='.cook') as root_tmp:
+ root = Path(root_tmp)
+ for revision in self.storage.revision_log([obj_id]):
+ revhex = hashutil.hash_to_hex(revision['id'])
+ revdir = (root / revhex)
+ revdir.mkdir()
+ directory_cooker.build_directory(revision['directory'],
+ str(revdir).encode())
+
+ tar_buffer = io.BytesIO()
+ tar = tarfile.open(fileobj=tar_buffer, mode='w')
+ tar.add(root_tmp, arcname=hashutil.hash_to_hex(obj_id))
+ bundle_content = tar_buffer.getbuffer()
+
+ # Cache the bundle
+ self.update_cache(obj_id, bundle_content)
+ # Make a notification that the bundle have been cooked
+ # NOT YET IMPLEMENTED see TODO in function.
+ self.notify_bundle_ready(
+ notif_data='Bundle %s ready' % hashutil.hash_to_hex(obj_id),
+ bundle_id=obj_id)
+
+ def notify_bundle_ready(self, notif_data, bundle_id):
+ # TODO plug this method with the notification method once
+ # done.
+ pass
+
+
class DirectoryCooker():
"""Creates a cooked directory from its sha1_git in the db.
@@ -124,8 +180,18 @@
# Create temporary folder to retrieve the files into.
root = bytes(tempfile.mkdtemp(prefix='directory.',
suffix='.cook'), 'utf8')
+ self.build_directory(dir_id, root)
+ # Use the created directory to make a bundle with the data as
+ # a compressed directory.
+ bundle_content = self._create_bundle_content(
+ root,
+ hashutil.hash_to_hex(dir_id))
+ return bundle_content
+
+ def build_directory(self, dir_id, root):
# Retrieve data from the database.
data = self.storage.directory_ls(dir_id, recursive=True)
+
# Split into files and directory data.
# TODO(seirl): also handle revision data.
data1, data2 = itertools.tee(data, 2)
@@ -136,13 +202,6 @@
self._create_tree(root, dir_data)
self._create_files(root, file_data)
- # Use the created directory to make a bundle with the data as
- # a compressed directory.
- bundle_content = self._create_bundle_content(
- root,
- hashutil.hash_to_hex(dir_id))
- return bundle_content
-
def _create_tree(self, root, directory_paths):
"""Create a directory tree from the given paths

File Metadata

Mime Type
text/plain
Expires
Thu, Jul 3, 3:27 PM (1 w, 20 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3219696

Event Timeline

About · Developer information · Legal