diff --git a/debian/control b/debian/control --- a/debian/control +++ b/debian/control @@ -26,6 +26,7 @@ Depends: python3-swh.core (>= 0.0.20~), python3-swh.model (>= 0.0.15~), python3-swh.storage (>= 0.0.83~), + python3-swh.vault (>= 0.0.1~), ${misc:Depends}, ${python3:Depends} Description: Software Heritage Web Applications diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,3 +1,4 @@ swh.core >= 0.0.20 swh.model >= 0.0.15 swh.storage >= 0.0.83 +swh.vault >= 0.0.1 diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -5,11 +5,11 @@ # Runtime dependencies django djangorestframework -python-dateutil docutils -pygments -yaml file_magic +pygments +python-dateutil +pyyaml #Doc dependencies sphinx diff --git a/swh/web/api/urls.py b/swh/web/api/urls.py --- a/swh/web/api/urls.py +++ b/swh/web/api/urls.py @@ -3,14 +3,15 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import swh.web.api.views.origin # noqa import swh.web.api.views.content # noqa +import swh.web.api.views.directory # noqa +import swh.web.api.views.entity # noqa +import swh.web.api.views.origin # noqa import swh.web.api.views.person # noqa import swh.web.api.views.release # noqa import swh.web.api.views.revision # noqa -import swh.web.api.views.directory # noqa -import swh.web.api.views.entity # noqa import swh.web.api.views.stat # noqa +import swh.web.api.views.vault # noqa from swh.web.api.apiurls import APIUrls diff --git a/swh/web/api/views/vault.py b/swh/web/api/views/vault.py new file mode 100644 --- /dev/null +++ b/swh/web/api/views/vault.py @@ -0,0 +1,138 @@ +# Copyright (C) 2015-2017 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from django.http import HttpResponse + +from swh.web.common import service, query +from swh.web.common.utils import reverse +from swh.web.api import apidoc as api_doc +from swh.web.api.apiurls import api_route +from swh.web.api.views.utils import ( + api_lookup, doc_exc_id_not_found, doc_exc_bad_id, +) + + +@api_route('/vault/directory/(?P[a-fA-F0-9]+)/', + 'vault-cook-directory') +@api_doc.route('/vault/directory', tags=['hidden']) +@api_doc.arg('dir_id', + default='d4a96ba891017d0d26c15e509b4e6515e40d75ee', + argtype=api_doc.argtypes.sha1_git, + argdoc="The directory's sha1 identifier") +@api_doc.param('email', default=None, + argtype=api_doc.argtypes.int, + doc="e-mail to notify when the bundle is ready") +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) +@api_doc.returns(rettype=api_doc.rettypes.dict, + retdoc=('dictionary mapping containing the status of ' + 'the cooking')) +def api_vault_cook_directory(request, dir_id): + """Requests an archive of the directoy identified by dir_id. + + To import the directory in the current directory, use:: + + $ tar xvf path/to/directory.tar.gz + """ + email = request.GET.get('email', None) + _, obj_id = query.parse_hash_with_algorithms_or_throws( + dir_id, ['sha1'], 'Only sha1_git is supported.') + + def _enrich_dir_cook(res): + res['fetch_url'] = reverse('vault-fetch-directory', + kwargs={'dir_id': dir_id}) + return res + + return api_lookup( + service.vault_cook, 'directory', obj_id, email, + notfound_msg="Directory with ID '{}' not found.".format(dir_id), + enrich_fn=_enrich_dir_cook) + + +@api_route(r'/vault/directory/(?P[a-fA-F0-9]+)/raw/', + 'vault-fetch-directory') +@api_doc.route('/vault/directory/raw', tags=['hidden'], handle_response=True) +@api_doc.arg('dir_id', + default='d4a96ba891017d0d26c15e509b4e6515e40d75ee', + argtype=api_doc.argtypes.sha1_git, + argdoc="The directory's sha1 identifier") +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) +@api_doc.returns(rettype=api_doc.rettypes.octet_stream, + retdoc='the cooked directory tarball') +def api_vault_fetch_directory(request, dir_id): + """Fetch the archive of the directoy identified by dir_id.""" + _, obj_id = query.parse_hash_with_algorithms_or_throws( + dir_id, ['sha1'], 'Only sha1_git is supported.') + res = api_lookup( + service.vault_fetch, 'directory', obj_id, + notfound_msg="Directory with ID '{}' not found.".format(dir_id)) + fname = '{}.tar.gz'.format(dir_id) + response = HttpResponse(res, content_type='application/gzip') + response['Content-disposition'] = 'attachment; filename={}'.format(fname) + return response + + +@api_route(r'/vault/revision_gitfast/(?P[a-fA-F0-9]+)/', + 'vault-cook-revision_gitfast') +@api_doc.route('/vault/revision_gitfast', tags=['hidden']) +@api_doc.arg('rev_id', + default='9174026cfe69d73ef80b27890615f8b2ef5c265a', + argtype=api_doc.argtypes.sha1_git, + argdoc="The revision's sha1_git identifier") +@api_doc.param('email', default=None, + argtype=api_doc.argtypes.int, + doc="e-mail to notify when the bundle is ready") +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) +@api_doc.returns(rettype=api_doc.rettypes.dict, + retdoc='dictionary mapping containing the status of ' + 'the cooking') +def api_vault_cook_revision_gitfast(request, rev_id): + """Requests an archive of the revision identified by rev_id. + + To import the revision in the current directory, use:: + + $ git init + $ zcat path/to/revision.gitfast.gz | git fast-import + """ + email = request.GET.get('email', None) + _, obj_id = query.parse_hash_with_algorithms_or_throws( + rev_id, ['sha1'], 'Only sha1_git is supported.') + + def _enrich_dir_cook(res): + res['fetch_url'] = reverse('vault-fetch-revision_gitfast', + kwargs={'rev_id': rev_id}) + return res + + return api_lookup( + service.vault_cook, 'revision_gitfast', obj_id, email, + notfound_msg="Revision with ID '{}' not found.".format(rev_id), + enrich_fn=_enrich_dir_cook) + + +@api_route('/vault/revision_gitfast/(?P[a-fA-F0-9]+)/raw/', + 'vault-fetch-revision_gitfast') +@api_doc.route('/vault/revision_gitfast/raw', tags=['hidden'], + handle_response=True) +@api_doc.arg('rev_id', + default='9174026cfe69d73ef80b27890615f8b2ef5c265a', + argtype=api_doc.argtypes.sha1_git, + argdoc="The revision's sha1_git identifier") +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) +@api_doc.returns(rettype=api_doc.rettypes.octet_stream, + retdoc='the cooked revision git fast-export') +def api_vault_fetch_revision_gitfast(request, rev_id): + """Fetch the archive of the revision identified by rev_id.""" + _, obj_id = query.parse_hash_with_algorithms_or_throws( + rev_id, ['sha1'], 'Only sha1_git is supported.') + res = api_lookup( + service.vault_fetch, 'revision_gitfast', obj_id, + notfound_msg="Revision with ID '{}' not found.".format(rev_id)) + fname = '{}.gitfast.gz'.format(rev_id) + response = HttpResponse(res, content_type='application/gzip') + response['Content-disposition'] = 'attachment; filename={}'.format(fname) + return response diff --git a/swh/web/common/service.py b/swh/web/common/service.py --- a/swh/web/common/service.py +++ b/swh/web/common/service.py @@ -15,6 +15,7 @@ from swh.web import config storage = config.storage() +vault = config.vault() MAX_LIMIT = 50 # Top limit the users can ask for @@ -826,3 +827,15 @@ raise NotFoundExc('Revision with criterion %s not found!' % revision) return (rev['id'], lookup_directory_with_revision(rev['id'], path, with_data)) + + +def vault_cook(obj_type, obj_id, email=None): + """Cook a vault bundle. + """ + return vault.cook(obj_type, obj_id, email=email) + + +def vault_fetch(obj_type, obj_id): + """Fetch a vault bundle. + """ + return vault.fetch(obj_type, obj_id) diff --git a/swh/web/config.py b/swh/web/config.py --- a/swh/web/config.py +++ b/swh/web/config.py @@ -5,14 +5,17 @@ from swh.core import config from swh.storage import get_storage +from swh.vault.api.client import RemoteVaultClient DEFAULT_CONFIG = { + 'allowed_hosts': ('list', []), 'storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5002/', }, }), + 'vault': ('string', 'http://127.0.0.1:5005/'), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', False), 'host': ('string', '127.0.0.1'), # development property @@ -30,7 +33,7 @@ }) } -swhweb_config = None +swhweb_config = {} def get_config(config_file='webapp/webapp'): @@ -39,11 +42,12 @@ dict. If no configuration file is provided, return a default configuration.""" - global swhweb_config if not swhweb_config: - swhweb_config = config.load_named_config(config_file, DEFAULT_CONFIG) + cfg = config.load_named_config(config_file, DEFAULT_CONFIG) + swhweb_config.update(cfg) config.prepare_folders(swhweb_config, 'log_dir') swhweb_config['storage'] = get_storage(**swhweb_config['storage']) + swhweb_config['vault'] = RemoteVaultClient(swhweb_config['vault']) return swhweb_config @@ -52,3 +56,10 @@ """ return get_config()['storage'] + + +def vault(): + """Return the current application's SWH vault. + + """ + return get_config()['vault'] diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py --- a/swh/web/settings/common.py +++ b/swh/web/settings/common.py @@ -35,7 +35,7 @@ DEBUG = swh_web_config['debug'] DEBUG_PROPAGATE_EXCEPTIONS = swh_web_config['debug'] -ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] +ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] + swh_web_config['allowed_hosts'] # Application definition diff --git a/swh/web/tests/api/views/test_vault.py b/swh/web/tests/api/views/test_vault.py new file mode 100644 --- /dev/null +++ b/swh/web/tests/api/views/test_vault.py @@ -0,0 +1,80 @@ +# Copyright (C) 2017 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from nose.tools import istest +from unittest.mock import patch + +from swh.model import hashutil + +from ..swh_api_testcase import SWHApiTestCase + +TEST_OBJ_ID = 'd4905454cc154b492bd6afed48694ae3c579345e' + + +class VaultApiTestCase(SWHApiTestCase): + @patch('swh.web.api.views.vault.service') + @istest + def api_vault_cook(self, mock_service): + stub_cook = { + 'fetch_url': ('http://127.0.0.1:5004/api/1/vault/directory/{}/raw/' + .format(TEST_OBJ_ID)), + 'obj_id': 'd4905454cc154b492bd6afed48694ae3c579345e', + 'obj_type': 'test_type', + 'progress_message': None, + 'status': 'done', + 'task_uuid': 'de75c902-5ee5-4739-996e-448376a93eff', + } + stub_fetch = b'content' + + mock_service.vault_cook.return_value = stub_cook + mock_service.vault_fetch.return_value = stub_fetch + + for obj_type in ('directory', 'revision_gitfast'): + rv = self.client.get(('/api/1/vault/{}/{}/?email=test@test.mail') + .format(obj_type, TEST_OBJ_ID)) + + self.assertEquals(rv.status_code, 200) + self.assertEquals(rv['Content-Type'], 'application/json') + + self.assertEquals(rv.data, stub_cook) + mock_service.vault_cook.assert_called_with( + obj_type, + hashutil.hash_to_bytes(TEST_OBJ_ID), + 'test@test.mail') + + rv = self.client.get(('/api/1/vault/{}/{}/raw/') + .format(obj_type, TEST_OBJ_ID)) + + self.assertEquals(rv.status_code, 200) + self.assertEquals(rv['Content-Type'], 'application/gzip') + self.assertEquals(rv.content, stub_fetch) + mock_service.vault_fetch.assert_called_with( + obj_type, hashutil.hash_to_bytes(TEST_OBJ_ID)) + + @patch('swh.web.api.views.vault.service') + @istest + def api_vault_cook_notfound(self, mock_service): + mock_service.vault_cook.return_value = None + mock_service.vault_fetch.return_value = None + + for obj_type in ('directory', 'revision_gitfast'): + rv = self.client.get(('/api/1/vault/{}/{}/') + .format(obj_type, TEST_OBJ_ID)) + + self.assertEquals(rv.status_code, 404) + self.assertEquals(rv['Content-Type'], 'application/json') + + self.assertEquals(rv.data['exception'], 'NotFoundExc') + mock_service.vault_cook.assert_called_with( + obj_type, hashutil.hash_to_bytes(TEST_OBJ_ID), None) + + rv = self.client.get(('/api/1/vault/{}/{}/raw/') + .format(obj_type, TEST_OBJ_ID)) + + self.assertEquals(rv.status_code, 404) + self.assertEquals(rv['Content-Type'], 'application/json') + self.assertEquals(rv.data['exception'], 'NotFoundExc') + mock_service.vault_fetch.assert_called_with( + obj_type, hashutil.hash_to_bytes(TEST_OBJ_ID))