diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,3 +1,4 @@ swh.core >= 0.0.20 swh.model >= 0.0.15 swh.storage >= 0.0.83 +swh.vault >= 0.0.1 diff --git a/swh/web/ui/backend.py b/swh/web/ui/backend.py --- a/swh/web/ui/backend.py +++ b/swh/web/ui/backend.py @@ -355,3 +355,15 @@ """ return main.storage().entity_get(uuid) + + +def vault_cook(obj_type, obj_id, email=None): + """Cook a vault bundle. + """ + return main.vault().cook(obj_type, obj_id, email=email) + + +def vault_fetch(obj_type, obj_id): + """Fetch a vault bundle. + """ + return main.vault().fetch(obj_type, obj_id) diff --git a/swh/web/ui/main.py b/swh/web/ui/main.py --- a/swh/web/ui/main.py +++ b/swh/web/ui/main.py @@ -17,6 +17,7 @@ from swh.web.ui.renderers import revision_id_from_url, highlight_source from swh.web.ui.renderers import SWHMultiResponse, urlize_header_links from swh.storage import get_storage +from swh.vault.api.client import RemoteVaultClient DEFAULT_CONFIG = { @@ -26,6 +27,7 @@ 'url': 'http://127.0.0.1:5002/', }, }), + 'vault': ('string', 'http://127.0.0.1:5005/'), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', None), 'host': ('string', '127.0.0.1'), @@ -64,6 +66,7 @@ conf = config.read(config_file, DEFAULT_CONFIG) config.prepare_folders(conf, 'log_dir') conf['storage'] = get_storage(**conf['storage']) + conf['vault'] = RemoteVaultClient(conf['vault']) return conf @@ -95,6 +98,12 @@ """ return app.config['conf']['storage'] +def vault(): + """Return the current application's storage. + + """ + return app.config['conf']['vault'] + def prepare_limiter(): """Prepare Flask Limiter from configuration and App configuration""" if hasattr(app, 'limiter'): diff --git a/swh/web/ui/service.py b/swh/web/ui/service.py --- a/swh/web/ui/service.py +++ b/swh/web/ui/service.py @@ -767,3 +767,11 @@ raise NotFoundExc('Revision with criterion %s not found!' % revision) return (rev['id'], lookup_directory_with_revision(rev['id'], path, with_data)) + + +def vault_cook(obj_type, obj_id, email=None): + return backend.vault_cook(obj_type, obj_id, email=email) + + +def vault_fetch(obj_type, obj_id): + return backend.vault_fetch(obj_type, obj_id) diff --git a/swh/web/ui/tests/views/test_api.py b/swh/web/ui/tests/views/test_api.py --- a/swh/web/ui/tests/views/test_api.py +++ b/swh/web/ui/tests/views/test_api.py @@ -2465,6 +2465,73 @@ mock_service.lookup_entity_by_uuid.assert_called_once_with( '34bd6b1b-463f-43e5-a697-785107f598e4') + @patch('swh.web.ui.views.api.service') + @istest + def api_vault_cook(self, mock_service): + stub_cook = { + 'fetch_url': ('http://127.0.0.1:5004/api/1/vault/directory/' + 'd4905454cc154b492bd6afed48694ae3c579345e/raw/'), + 'obj_id': 'd4905454cc154b492bd6afed48694ae3c579345e', + 'obj_type': 'test_type', + 'progress_message': None, + 'status': 'done', + 'task_uuid': 'de75c902-5ee5-4739-996e-448376a93eff', + } + stub_fetch = b'content' + + mock_service.vault_cook.return_value = stub_cook + mock_service.vault_fetch.return_value = stub_fetch + + for obj_type in ('directory', 'revision_gitfast'): + rv = self.app.get(('/api/1/vault/{}/' + 'd4905454cc154b492bd6afed48694ae3c579345e/' + '?email=test@test.mail') + .format(obj_type)) + + self.assertEquals(rv.status_code, 200) + self.assertEquals(rv.mimetype, 'application/json') + + response_data = json.loads(rv.data.decode('utf-8')) + self.assertEquals(response_data, stub_cook) + mock_service.vault_cook.assert_called_with( + obj_type, 'd4905454cc154b492bd6afed48694ae3c579345e', + 'test@test.mail') + + rv = self.app.get(('/api/1/vault/{}/' + 'd4905454cc154b492bd6afed48694ae3c579345e/raw/') + .format(obj_type)) + + self.assertEquals(rv.status_code, 200) + self.assertEquals(rv.mimetype, 'application/gzip') + self.assertEquals(rv.data, stub_fetch) + mock_service.vault_fetch.assert_called_with( + obj_type, 'd4905454cc154b492bd6afed48694ae3c579345e') + + @patch('swh.web.ui.views.api.service') + @istest + def api_vault_cook_notfound(self, mock_service): + mock_service.vault_cook.return_value = None + mock_service.vault_fetch.return_value = None + + for obj_type in ('directory', 'revision_gitfast'): + rv = self.app.get(('/api/1/vault/{}/4040/') + .format(obj_type)) + + self.assertEquals(rv.status_code, 404) + self.assertEquals(rv.mimetype, 'application/json') + + response_data = json.loads(rv.data.decode('utf-8')) + self.assertEquals(response_data['exception'], 'NotFoundExc') + mock_service.vault_cook.assert_called_with(obj_type, '4040', None) + + rv = self.app.get(('/api/1/vault/{}/4040/raw/') + .format(obj_type)) + + self.assertEquals(rv.status_code, 404) + self.assertEquals(rv.mimetype, 'application/json') + self.assertEquals(response_data['exception'], 'NotFoundExc') + mock_service.vault_fetch.assert_called_with(obj_type, '4040') + class ApiUtils(unittest.TestCase): diff --git a/swh/web/ui/views/api.py b/swh/web/ui/views/api.py --- a/swh/web/ui/views/api.py +++ b/swh/web/ui/views/api.py @@ -1103,5 +1103,119 @@ """ return _api_lookup( service.lookup_entity_by_uuid, uuid, - notfound_msg="Entity with uuid '%s' not found." % uuid, + notfound_msg="Entity with uuid '{}' not found.".format(uuid), enrich_fn=utils.enrich_entity) + + +@app.route('/api/1/vault/directory//', + methods=['GET', 'POST']) +@doc.route('/api/1/vault/directory', tags=['hidden']) +@doc.arg('dir_id', + default='', + argtype=doc.argtypes.sha1_git, + argdoc="The directory's sha1 identifier") +@doc.param('email', default=None, + argtype=doc.argtypes.int, + doc="e-mail to notify when the bundle is ready") +@doc.raises(exc=doc.excs.badinput, doc=_doc_exc_bad_id) +@doc.raises(exc=doc.excs.notfound, doc=_doc_exc_id_not_found) +@doc.returns(rettype=doc.rettypes.dict, + retdoc='dictionary mapping containing the status of the cooking') +def api_vault_cook_directory(dir_id): + """Requests an archive of the directoy identified by dir_id. + + To import the directory in the current directory, use:: + + $ tar xvf path/to/directory.tar.gz + """ + email = request.values.get('email', None) + + def _enrich_dir_cook(res): + res['fetch_url'] = url_for('api_vault_fetch_directory', dir_id=dir_id) + return res + + return _api_lookup( + service.vault_cook, 'directory', dir_id, email, + notfound_msg="Directory with ID '{}' not found.".format(dir_id), + enrich_fn=_enrich_dir_cook) + + +@app.route('/api/1/vault/directory//raw/') +@doc.route('/api/1/vault/directory/raw', tags=['hidden']) +@doc.arg('dir_id', + default='', + argtype=doc.argtypes.sha1_git, + argdoc="The directory's sha1 identifier") +@doc.raises(exc=doc.excs.badinput, doc=_doc_exc_bad_id) +@doc.raises(exc=doc.excs.notfound, doc=_doc_exc_id_not_found) +@doc.returns(rettype=doc.rettypes.octet_stream, + retdoc='the cooked directory tarball') +def api_vault_fetch_directory(dir_id): + """Fetch the archive of the directoy identified by dir_id.""" + res = _api_lookup( + service.vault_fetch, 'directory', dir_id, + notfound_msg="Directory with ID '{}' not found.".format(dir_id)) + filename = '{}.tar.gz'.format(dir_id) + + return app.response_class(res, + headers={'Content-disposition': 'attachment;' + 'filename=%s' % filename}, + mimetype='application/gzip') + + +@app.route('/api/1/vault/revision_gitfast//', + methods=['GET', 'POST']) +@doc.route('/api/1/vault/revision_gitfast', tags=['hidden']) +@doc.arg('rev_id', + default='', + argtype=doc.argtypes.sha1_git, + argdoc="The revision's sha1_git identifier") +@doc.param('email', default=None, + argtype=doc.argtypes.int, + doc="e-mail to notify when the bundle is ready") +@doc.raises(exc=doc.excs.badinput, doc=_doc_exc_bad_id) +@doc.raises(exc=doc.excs.notfound, doc=_doc_exc_id_not_found) +@doc.returns(rettype=doc.rettypes.dict, + retdoc='dictionary mapping containing the status of the cooking') +def api_vault_cook_revision_gitfast(rev_id): + """Requests an archive of the revision identified by rev_id. + + To import the revision in the current directory, use:: + + $ git init + $ zcat path/to/revision.gitfast.gz | git fast-import + """ + email = request.values.get('email', None) + + def _enrich_dir_cook(res): + res['fetch_url'] = url_for('api_vault_fetch_revision_gitfast', + rev_id=rev_id) + return res + + return _api_lookup( + service.vault_cook, 'revision_gitfast', rev_id, email, + notfound_msg="Revision with ID '{}' not found.".format(rev_id), + enrich_fn=_enrich_dir_cook) + + +@app.route('/api/1/vault/revision_gitfast//raw/') +@doc.route('/api/1/vault/revision_gitfast/raw', tags=['hidden']) +@doc.arg('rev_id', + default='', + argtype=doc.argtypes.sha1_git, + argdoc="The revision's sha1_git identifier") +@doc.raises(exc=doc.excs.badinput, doc=_doc_exc_bad_id) +@doc.raises(exc=doc.excs.notfound, doc=_doc_exc_id_not_found) +@doc.returns(rettype=doc.rettypes.octet_stream, + retdoc='the cooked revision git fast-export') +def api_vault_fetch_revision_gitfast(rev_id): + """Fetch the archive of the revision identified by rev_id.""" + res = _api_lookup( + service.vault_fetch, 'revision_gitfast', rev_id, + notfound_msg="Revision with ID '{}' not found.".format(rev_id)) + filename = '{}.gitfast.gz'.format(rev_id) + + return app.response_class(res, + headers={'Content-disposition': 'attachment;' + 'filename=%s' % filename}, + mimetype='application/gzip')