Page MenuHomeSoftware Heritage
Files F7066125

D264.diff
No OneTemporary
Actions

D264.diff
View Options

diff --git a/debian/control b/debian/control
--- a/debian/control
+++ b/debian/control
@@ -26,6 +26,7 @@
Depends: python3-swh.core (>= 0.0.20~),
python3-swh.model (>= 0.0.15~),
python3-swh.storage (>= 0.0.83~),
+ python3-swh.vault (>= 0.0.1~),
${misc:Depends},
${python3:Depends}
Description: Software Heritage Web Applications
diff --git a/requirements-swh.txt b/requirements-swh.txt
--- a/requirements-swh.txt
+++ b/requirements-swh.txt
@@ -1,3 +1,4 @@
swh.core >= 0.0.20
swh.model >= 0.0.15
swh.storage >= 0.0.83
+swh.vault >= 0.0.1
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,11 +5,11 @@
# Runtime dependencies
django
djangorestframework
-python-dateutil
docutils
-pygments
-yaml
file_magic
+pygments
+python-dateutil
+pyyaml
#Doc dependencies
sphinx
diff --git a/swh/web/api/urls.py b/swh/web/api/urls.py
--- a/swh/web/api/urls.py
+++ b/swh/web/api/urls.py
@@ -3,14 +3,15 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-import swh.web.api.views.origin # noqa
import swh.web.api.views.content # noqa
+import swh.web.api.views.directory # noqa
+import swh.web.api.views.entity # noqa
+import swh.web.api.views.origin # noqa
import swh.web.api.views.person # noqa
import swh.web.api.views.release # noqa
import swh.web.api.views.revision # noqa
-import swh.web.api.views.directory # noqa
-import swh.web.api.views.entity # noqa
import swh.web.api.views.stat # noqa
+import swh.web.api.views.vault # noqa
from swh.web.api.apiurls import APIUrls
diff --git a/swh/web/api/views/vault.py b/swh/web/api/views/vault.py
new file mode 100644
--- /dev/null
+++ b/swh/web/api/views/vault.py
@@ -0,0 +1,138 @@
+# Copyright (C) 2015-2017 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU Affero General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from django.http import HttpResponse
+
+from swh.web.common import service, query
+from swh.web.common.utils import reverse
+from swh.web.api import apidoc as api_doc
+from swh.web.api.apiurls import api_route
+from swh.web.api.views.utils import (
+ api_lookup, doc_exc_id_not_found, doc_exc_bad_id,
+)
+
+
+@api_route('/vault/directory/(?P<dir_id>[a-fA-F0-9]+)/',
+ 'vault-cook-directory')
+@api_doc.route('/vault/directory', tags=['hidden'])
+@api_doc.arg('dir_id',
+ default='d4a96ba891017d0d26c15e509b4e6515e40d75ee',
+ argtype=api_doc.argtypes.sha1_git,
+ argdoc="The directory's sha1 identifier")
+@api_doc.param('email', default=None,
+ argtype=api_doc.argtypes.int,
+ doc="e-mail to notify when the bundle is ready")
+@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id)
+@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found)
+@api_doc.returns(rettype=api_doc.rettypes.dict,
+ retdoc=('dictionary mapping containing the status of '
+ 'the cooking'))
+def api_vault_cook_directory(request, dir_id):
+ """Requests an archive of the directoy identified by dir_id.
+
+ To import the directory in the current directory, use::
+
+ $ tar xvf path/to/directory.tar.gz
+ """
+ email = request.GET.get('email', None)
+ _, obj_id = query.parse_hash_with_algorithms_or_throws(
+ dir_id, ['sha1'], 'Only sha1_git is supported.')
+
+ def _enrich_dir_cook(res):
+ res['fetch_url'] = reverse('vault-fetch-directory',
+ kwargs={'dir_id': dir_id})
+ return res
+
+ return api_lookup(
+ service.vault_cook, 'directory', obj_id, email,
+ notfound_msg="Directory with ID '{}' not found.".format(dir_id),
+ enrich_fn=_enrich_dir_cook)
+
+
+@api_route(r'/vault/directory/(?P<dir_id>[a-fA-F0-9]+)/raw/',
+ 'vault-fetch-directory')
+@api_doc.route('/vault/directory/raw', tags=['hidden'], handle_response=True)
+@api_doc.arg('dir_id',
+ default='d4a96ba891017d0d26c15e509b4e6515e40d75ee',
+ argtype=api_doc.argtypes.sha1_git,
+ argdoc="The directory's sha1 identifier")
+@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id)
+@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found)
+@api_doc.returns(rettype=api_doc.rettypes.octet_stream,
+ retdoc='the cooked directory tarball')
+def api_vault_fetch_directory(request, dir_id):
+ """Fetch the archive of the directoy identified by dir_id."""
+ _, obj_id = query.parse_hash_with_algorithms_or_throws(
+ dir_id, ['sha1'], 'Only sha1_git is supported.')
+ res = api_lookup(
+ service.vault_fetch, 'directory', obj_id,
+ notfound_msg="Directory with ID '{}' not found.".format(dir_id))
+ fname = '{}.tar.gz'.format(dir_id)
+ response = HttpResponse(res, content_type='application/gzip')
+ response['Content-disposition'] = 'attachment; filename={}'.format(fname)
+ return response
+
+
+@api_route(r'/vault/revision_gitfast/(?P<rev_id>[a-fA-F0-9]+)/',
+ 'vault-cook-revision_gitfast')
+@api_doc.route('/vault/revision_gitfast', tags=['hidden'])
+@api_doc.arg('rev_id',
+ default='9174026cfe69d73ef80b27890615f8b2ef5c265a',
+ argtype=api_doc.argtypes.sha1_git,
+ argdoc="The revision's sha1_git identifier")
+@api_doc.param('email', default=None,
+ argtype=api_doc.argtypes.int,
+ doc="e-mail to notify when the bundle is ready")
+@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id)
+@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found)
+@api_doc.returns(rettype=api_doc.rettypes.dict,
+ retdoc='dictionary mapping containing the status of '
+ 'the cooking')
+def api_vault_cook_revision_gitfast(request, rev_id):
+ """Requests an archive of the revision identified by rev_id.
+
+ To import the revision in the current directory, use::
+
+ $ git init
+ $ zcat path/to/revision.gitfast.gz | git fast-import
+ """
+ email = request.GET.get('email', None)
+ _, obj_id = query.parse_hash_with_algorithms_or_throws(
+ rev_id, ['sha1'], 'Only sha1_git is supported.')
+
+ def _enrich_dir_cook(res):
+ res['fetch_url'] = reverse('vault-fetch-revision_gitfast',
+ kwargs={'rev_id': rev_id})
+ return res
+
+ return api_lookup(
+ service.vault_cook, 'revision_gitfast', obj_id, email,
+ notfound_msg="Revision with ID '{}' not found.".format(rev_id),
+ enrich_fn=_enrich_dir_cook)
+
+
+@api_route('/vault/revision_gitfast/(?P<rev_id>[a-fA-F0-9]+)/raw/',
+ 'vault-fetch-revision_gitfast')
+@api_doc.route('/vault/revision_gitfast/raw', tags=['hidden'],
+ handle_response=True)
+@api_doc.arg('rev_id',
+ default='9174026cfe69d73ef80b27890615f8b2ef5c265a',
+ argtype=api_doc.argtypes.sha1_git,
+ argdoc="The revision's sha1_git identifier")
+@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id)
+@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found)
+@api_doc.returns(rettype=api_doc.rettypes.octet_stream,
+ retdoc='the cooked revision git fast-export')
+def api_vault_fetch_revision_gitfast(request, rev_id):
+ """Fetch the archive of the revision identified by rev_id."""
+ _, obj_id = query.parse_hash_with_algorithms_or_throws(
+ rev_id, ['sha1'], 'Only sha1_git is supported.')
+ res = api_lookup(
+ service.vault_fetch, 'revision_gitfast', obj_id,
+ notfound_msg="Revision with ID '{}' not found.".format(rev_id))
+ fname = '{}.gitfast.gz'.format(rev_id)
+ response = HttpResponse(res, content_type='application/gzip')
+ response['Content-disposition'] = 'attachment; filename={}'.format(fname)
+ return response
diff --git a/swh/web/common/service.py b/swh/web/common/service.py
--- a/swh/web/common/service.py
+++ b/swh/web/common/service.py
@@ -15,6 +15,7 @@
from swh.web import config
storage = config.storage()
+vault = config.vault()
MAX_LIMIT = 50 # Top limit the users can ask for
@@ -826,3 +827,15 @@
raise NotFoundExc('Revision with criterion %s not found!' % revision)
return (rev['id'],
lookup_directory_with_revision(rev['id'], path, with_data))
+
+
+def vault_cook(obj_type, obj_id, email=None):
+ """Cook a vault bundle.
+ """
+ return vault.cook(obj_type, obj_id, email=email)
+
+
+def vault_fetch(obj_type, obj_id):
+ """Fetch a vault bundle.
+ """
+ return vault.fetch(obj_type, obj_id)
diff --git a/swh/web/config.py b/swh/web/config.py
--- a/swh/web/config.py
+++ b/swh/web/config.py
@@ -5,14 +5,17 @@
from swh.core import config
from swh.storage import get_storage
+from swh.vault.api.client import RemoteVaultClient
DEFAULT_CONFIG = {
+ 'allowed_hosts': ('list', []),
'storage': ('dict', {
'cls': 'remote',
'args': {
'url': 'http://127.0.0.1:5002/',
},
}),
+ 'vault': ('string', 'http://127.0.0.1:5005/'),
'log_dir': ('string', '/tmp/swh/log'),
'debug': ('bool', False),
'host': ('string', '127.0.0.1'), # development property
@@ -30,7 +33,7 @@
})
}
-swhweb_config = None
+swhweb_config = {}
def get_config(config_file='webapp/webapp'):
@@ -39,11 +42,12 @@
dict. If no configuration file is provided, return a default
configuration."""
- global swhweb_config
if not swhweb_config:
- swhweb_config = config.load_named_config(config_file, DEFAULT_CONFIG)
+ cfg = config.load_named_config(config_file, DEFAULT_CONFIG)
+ swhweb_config.update(cfg)
config.prepare_folders(swhweb_config, 'log_dir')
swhweb_config['storage'] = get_storage(**swhweb_config['storage'])
+ swhweb_config['vault'] = RemoteVaultClient(swhweb_config['vault'])
return swhweb_config
@@ -52,3 +56,10 @@
"""
return get_config()['storage']
+
+
+def vault():
+ """Return the current application's SWH vault.
+
+ """
+ return get_config()['vault']
diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py
--- a/swh/web/settings/common.py
+++ b/swh/web/settings/common.py
@@ -35,7 +35,7 @@
DEBUG = swh_web_config['debug']
DEBUG_PROPAGATE_EXCEPTIONS = swh_web_config['debug']
-ALLOWED_HOSTS = ['127.0.0.1', 'localhost']
+ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] + swh_web_config['allowed_hosts']
# Application definition
diff --git a/swh/web/tests/api/views/test_vault.py b/swh/web/tests/api/views/test_vault.py
new file mode 100644
--- /dev/null
+++ b/swh/web/tests/api/views/test_vault.py
@@ -0,0 +1,80 @@
+# Copyright (C) 2017 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU Affero General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from nose.tools import istest
+from unittest.mock import patch
+
+from swh.model import hashutil
+
+from ..swh_api_testcase import SWHApiTestCase
+
+TEST_OBJ_ID = 'd4905454cc154b492bd6afed48694ae3c579345e'
+
+
+class VaultApiTestCase(SWHApiTestCase):
+ @patch('swh.web.api.views.vault.service')
+ @istest
+ def api_vault_cook(self, mock_service):
+ stub_cook = {
+ 'fetch_url': ('http://127.0.0.1:5004/api/1/vault/directory/{}/raw/'
+ .format(TEST_OBJ_ID)),
+ 'obj_id': 'd4905454cc154b492bd6afed48694ae3c579345e',
+ 'obj_type': 'test_type',
+ 'progress_message': None,
+ 'status': 'done',
+ 'task_uuid': 'de75c902-5ee5-4739-996e-448376a93eff',
+ }
+ stub_fetch = b'content'
+
+ mock_service.vault_cook.return_value = stub_cook
+ mock_service.vault_fetch.return_value = stub_fetch
+
+ for obj_type in ('directory', 'revision_gitfast'):
+ rv = self.client.get(('/api/1/vault/{}/{}/?email=test@test.mail')
+ .format(obj_type, TEST_OBJ_ID))
+
+ self.assertEquals(rv.status_code, 200)
+ self.assertEquals(rv['Content-Type'], 'application/json')
+
+ self.assertEquals(rv.data, stub_cook)
+ mock_service.vault_cook.assert_called_with(
+ obj_type,
+ hashutil.hash_to_bytes(TEST_OBJ_ID),
+ 'test@test.mail')
+
+ rv = self.client.get(('/api/1/vault/{}/{}/raw/')
+ .format(obj_type, TEST_OBJ_ID))
+
+ self.assertEquals(rv.status_code, 200)
+ self.assertEquals(rv['Content-Type'], 'application/gzip')
+ self.assertEquals(rv.content, stub_fetch)
+ mock_service.vault_fetch.assert_called_with(
+ obj_type, hashutil.hash_to_bytes(TEST_OBJ_ID))
+
+ @patch('swh.web.api.views.vault.service')
+ @istest
+ def api_vault_cook_notfound(self, mock_service):
+ mock_service.vault_cook.return_value = None
+ mock_service.vault_fetch.return_value = None
+
+ for obj_type in ('directory', 'revision_gitfast'):
+ rv = self.client.get(('/api/1/vault/{}/{}/')
+ .format(obj_type, TEST_OBJ_ID))
+
+ self.assertEquals(rv.status_code, 404)
+ self.assertEquals(rv['Content-Type'], 'application/json')
+
+ self.assertEquals(rv.data['exception'], 'NotFoundExc')
+ mock_service.vault_cook.assert_called_with(
+ obj_type, hashutil.hash_to_bytes(TEST_OBJ_ID), None)
+
+ rv = self.client.get(('/api/1/vault/{}/{}/raw/')
+ .format(obj_type, TEST_OBJ_ID))
+
+ self.assertEquals(rv.status_code, 404)
+ self.assertEquals(rv['Content-Type'], 'application/json')
+ self.assertEquals(rv.data['exception'], 'NotFoundExc')
+ mock_service.vault_fetch.assert_called_with(
+ obj_type, hashutil.hash_to_bytes(TEST_OBJ_ID))

File Metadata

Mime Type
text/plain
Expires
Nov 4 2024, 5:50 PM (9 w, 5 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3218500

Event Timeline

About · Developer information · Legal