diff --git a/swh/web/ui/renderers.py b/swh/web/ui/renderers.py index 140a1f0e..d61e1544 100644 --- a/swh/web/ui/renderers.py +++ b/swh/web/ui/renderers.py @@ -1,207 +1,239 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import re import yaml import json +from docutils.core import publish_parts +from docutils.writers.html4css1 import Writer, HTMLTranslator + from flask import make_response, request, Response, render_template from flask import g from flask.ext.api import renderers, parsers from flask_api.mediatypes import MediaType + from swh.web.ui import utils class SWHFilterEnricher(): """Global filter on fields. """ def filter_by_fields(self, data): """Extract a request parameter 'fields' if it exists to permit the filtering on the data dict's keys. If such field is not provided, returns the data as is. """ fields = request.args.get('fields') if fields: fields = set(fields.split(',')) data = utils.filter_field_keys(data, fields) return data class YAMLRenderer(renderers.BaseRenderer, SWHFilterEnricher): """Renderer for application/yaml. Orchestrate from python data structure to yaml. """ media_type = 'application/yaml' def render(self, data, media_type, **options): data = self.filter_by_fields(data) return yaml.dump(data, encoding=self.charset) class JSONPEnricher(): """JSONP rendering. """ def enrich_with_jsonp(self, data): """Defines a jsonp function that extracts a potential 'callback' request parameter holding the function name and wraps the data inside a call to such function e.g: GET /blah/foo/bar renders: {'output': 'wrapped'} GET /blah/foo/bar?callback=fn renders: fn({'output': 'wrapped'}) """ jsonp = request.args.get('callback') if jsonp: return '%s(%s)' % (jsonp, data) return data class SWHJSONRenderer(renderers.JSONRenderer, SWHFilterEnricher, JSONPEnricher): """Renderer for application/json. Serializes in json the data and returns it. Also deals with jsonp. If callback is found in request parameter, wrap the result as a function with name the value of the parameter query 'callback'. """ def render(self, data, media_type, **options): data = self.filter_by_fields(data) res = super().render(data, media_type, **options) return self.enrich_with_jsonp(res) class SWHMultiResponse(Response, SWHFilterEnricher): """ A Flask Response subclass. Override force_type to transform dict responses into callable Flask response objects whose mimetype matches the request's Accept header: HTML template render, YAML dump or default to a JSON dump. """ @classmethod def make_response_from_mimetype(cls, rv): def wants_html(best_match): return best_match == 'text/html' and \ request.accept_mimetypes[best_match] > \ request.accept_mimetypes['application/json'] def wants_yaml(best_match): return best_match == 'application/yaml' and \ request.accept_mimetypes[best_match] > \ request.accept_mimetypes['application/json'] if isinstance(rv, dict) or isinstance(rv, list): rv = cls.filter_by_fields(cls, rv) acc_mime = ['application/json', 'application/yaml', 'text/html'] best_match = request.accept_mimetypes.best_match(acc_mime) # return a template render if wants_html(best_match): data = json.dumps(rv, sort_keys=True, indent=4, separators=(',', ': ')) env = g.get('doc_env', {}) env['response_data'] = data env['request'] = request rv = Response(render_template('apidoc.html', **env), content_type='text/html') # return formatted yaml elif wants_yaml(best_match): rv = Response( yaml.dump(rv), content_type='application/yaml') # return formatted json else: # jsonify is unhappy with lists in Flask 0.10.1, use json.dumps rv = Response( json.dumps(rv), content_type='application/json') return rv @classmethod def force_type(cls, rv, environ=None): # Data from apidoc if isinstance(rv, dict) or isinstance(rv, list): rv = cls.make_response_from_mimetype(rv) return super().force_type(rv, environ) def urlize_api_links(content): """Utility function for decorating api links in browsable api.""" return re.sub(r'"(/api/.*|/browse/.*)"', r'"\1"', content) +class NoHeaderHTMLTranslator(HTMLTranslator): + """ + Docutils translator subclass to customize the generation of HTML + from reST-formatted docstrings + """ + def __init__(self, document): + super().__init__(document) + self.body_prefix = [] + self.body_suffix = [] + + # disable blockquotes to ignore indentation issue with docstrings + def visit_block_quote(self, node): + pass + + def depart_block_quote(self, node): + pass + + def visit_bullet_list(self, node): + self.context.append((self.compact_simple, self.compact_p)) + self.compact_p = None + self.compact_simple = self.is_compactable(node) + self.body.append(self.starttag(node, 'ul', CLASS='docstring')) + +DOCSTRING_WRITER = Writer() +DOCSTRING_WRITER.translator_class = NoHeaderHTMLTranslator + + def safe_docstring_display(docstring): - """Utility function to safely decorate docstring in browsable api.""" - src = r'(Args|Raises?|Throws?|Yields?|Returns?|Examples?|Samples?):.*' - dest = r'

\1:

  ' - return re.sub(src, dest, docstring) + """ + Utility function to htmlize reST-formatted documentation in browsable + api. + """ + return publish_parts(docstring, writer=DOCSTRING_WRITER)['html_body'] def revision_id_from_url(url): """Utility function to obtain a revision's ID from its browsing URL.""" return re.sub(r'/browse/revision/([0-9a-f]{40}|[0-9a-f]{64})/.*', r'\1', url) class SWHBrowsableAPIRenderer(renderers.BrowsableAPIRenderer): """SWH's browsable api renderer. """ template = "api.html" RENDERERS = [ 'swh.web.ui.renderers.SWHJSONRenderer', 'swh.web.ui.renderers.SWHBrowsableAPIRenderer', 'flask.ext.api.parsers.URLEncodedParser', 'swh.web.ui.renderers.YAMLRenderer', ] RENDERERS_INSTANCE = [ SWHJSONRenderer(), SWHBrowsableAPIRenderer(), parsers.URLEncodedParser(), YAMLRenderer(), ] RENDERERS_BY_TYPE = { r.media_type: r for r in RENDERERS_INSTANCE } def error_response(default_error_msg, error_code, error): """Private function to create a custom error response. """ # if nothing is requested by client, use json default_application_type = 'application/json' accept_type = request.headers.get('Accept', default_application_type) renderer = RENDERERS_BY_TYPE.get( accept_type, RENDERERS_BY_TYPE[default_application_type]) # for edge cases, use the elected renderer's media type accept_type = renderer.media_type response = make_response(default_error_msg, error_code) response.headers['Content-Type'] = accept_type response.data = renderer.render({"error": str(error)}, media_type=MediaType(accept_type), status=error_code, headers={'Content-Type': accept_type}) return response diff --git a/swh/web/ui/templates/apidoc.html b/swh/web/ui/templates/apidoc.html index 2c8c5cca..18ba24f3 100644 --- a/swh/web/ui/templates/apidoc.html +++ b/swh/web/ui/templates/apidoc.html @@ -1,84 +1,84 @@ {% extends "layout.html" %} {% block title %}Software Heritage API{% endblock %} {% block content %} {% if docstring %}

Overview

- {{ docstring | safe }} + {% autoescape off %} {{ docstring | safe_docstring_display }} {% endautoescape %}
{% endif %} {% if response_data and response_data is not none %}

Request 

{{ request.method }} {{ request.url }}

Result 

 {% autoescape off %} {{ response_data | urlize_api_links }} {% endautoescape %}
{% endif %}
{% for url in urls %} {% endfor %}
URL Allowed Methods
{{ url['rule'] }} {{ url['methods'] | sort | join(', ') }}
{% if args and args|length > 0 %}

Args

{% for arg in args %}
{{ arg['name'] }}: {{ arg['type'] }}
-
{{ arg['doc'] }}
+
{% autoescape off %} {{ arg['doc'] | safe_docstring_display }} {% endautoescape %}
{% endfor %}
-
+ {% endif %} {% if excs and excs|length > 0 %}

Raises

{% for exc in excs %}
{{ exc['exc'] }}
-
{{ exc['doc'] }}
+
{% autoescape off %} {{ exc['doc'] | safe_docstring_display }} {% endautoescape %}
{% endfor %}
{% endif %} {% if return %}

Returns

-
{{ return['type'] }}
-
{{ return['doc'] }}
+
{{ return['type'] }}
+
{% autoescape off %} {{ return['doc'] | safe_docstring_display }} {% endautoescape %}
{% endif %} {% if example %}

Example

{{ example }}
{% endif %} {% endblock %} diff --git a/swh/web/ui/tests/test_renderers.py b/swh/web/ui/tests/test_renderers.py index 5a43d916..e78f6681 100644 --- a/swh/web/ui/tests/test_renderers.py +++ b/swh/web/ui/tests/test_renderers.py @@ -1,364 +1,356 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import unittest import yaml from flask import Response from flask_api.mediatypes import MediaType from nose.tools import istest from unittest.mock import patch, MagicMock from swh.web.ui import renderers class RendererTestCase(unittest.TestCase): @patch('swh.web.ui.renderers.g') @patch('swh.web.ui.renderers.json') @patch('swh.web.ui.renderers.request') @patch('swh.web.ui.renderers.render_template') @patch('swh.web.ui.renderers.SWHMultiResponse.filter_by_fields') @istest def swh_multi_response_mimetype_html(self, mock_filter, mock_render, mock_request, mock_json, mock_g): # given data = {'data': [12, 34], 'id': 'adc83b19e793491b1c6ea0fd8b46cd9f32e592fc'} mock_g.get.return_value = {'my_key': 'my_display_value'} mock_filter.return_value = data expected_env = { 'my_key': 'my_display_value', 'response_data': json.dumps(data), 'request': mock_request } def mock_mimetypes(key): mimetypes = { 'text/html': 10, 'application/json': 0.1, 'application/yaml': 0.1 } return mimetypes[key] accept_mimetypes = MagicMock() accept_mimetypes.__getitem__.side_effect = mock_mimetypes accept_mimetypes.best_match = MagicMock(return_value='text/html') mock_request.accept_mimetypes = accept_mimetypes mock_json.dumps.return_value = json.dumps(data) # when rv = renderers.SWHMultiResponse.make_response_from_mimetype(data) # then mock_filter.assert_called_once_with(renderers.SWHMultiResponse, data) mock_render.assert_called_with('apidoc.html', **expected_env) self.assertEqual(rv.status_code, 200) self.assertEqual(rv.mimetype, 'text/html') @patch('swh.web.ui.renderers.g') @patch('swh.web.ui.renderers.yaml') @patch('swh.web.ui.renderers.request') @patch('swh.web.ui.renderers.SWHMultiResponse.filter_by_fields') @istest def swh_multi_response_mimetype_yaml(self, mock_filter, mock_request, mock_yaml, mock_g): # given data = {'data': [12, 34], 'id': 'adc83b19e793491b1c6ea0fd8b46cd9f32e592fc'} def mock_mimetypes(key): mimetypes = { 'application/yaml': 10, 'application/json': 0.1, 'text/html': 0.1 } return mimetypes[key] accept_mimetypes = MagicMock() accept_mimetypes.__getitem__.side_effect = mock_mimetypes accept_mimetypes.best_match = MagicMock( return_value='application/yaml') mock_request.accept_mimetypes = accept_mimetypes mock_yaml.dump.return_value = yaml.dump(data) mock_filter.return_value = data # when rv = renderers.SWHMultiResponse.make_response_from_mimetype(data) # then mock_filter.assert_called_once_with(renderers.SWHMultiResponse, data) mock_yaml.dump.assert_called_once_with(data) self.assertEqual(rv.status_code, 200) self.assertEqual(rv.mimetype, 'application/yaml') self.assertEqual(data, yaml.load(rv.data.decode('utf-8'))) @patch('swh.web.ui.renderers.g') @patch('swh.web.ui.renderers.json') @patch('swh.web.ui.renderers.request') @patch('swh.web.ui.renderers.SWHMultiResponse.filter_by_fields') @istest def swh_multi_response_mimetype_json(self, mock_filter, mock_request, mock_json, mock_g): # given data = {'data': [12, 34], 'id': 'adc83b19e793491b1c6ea0fd8b46cd9f32e592fc'} def mock_mimetypes(key): mimetypes = { 'application/json': 10, 'text/html': 0.1, 'application/yaml': 0.1 } return mimetypes[key] accept_mimetypes = MagicMock() accept_mimetypes.__getitem__.side_effect = mock_mimetypes accept_mimetypes.best_match = MagicMock( return_value='application/json') mock_request.accept_mimetypes = accept_mimetypes mock_json.dumps.return_value = json.dumps(data) mock_filter.return_value = data # when rv = renderers.SWHMultiResponse.make_response_from_mimetype(data) # then mock_filter.assert_called_once_with(renderers.SWHMultiResponse, data) mock_json.dumps.assert_called_once_with(data) self.assertEqual(rv.status_code, 200) self.assertEqual(rv.mimetype, 'application/json') self.assertEqual(data, json.loads(rv.data.decode('utf-8'))) @istest def apidoc_make_response_not_list_dict(self): # given incoming = Response() # when rv = renderers.SWHMultiResponse.make_response_from_mimetype(incoming) # then self.assertEqual(rv, incoming) @patch('swh.web.ui.renderers.request') @istest def swh_filter_renderer_do_nothing(self, mock_request): # given mock_request.args = {} swh_filter_renderer = renderers.SWHFilterEnricher() input_data = {'a': 'some-data'} # when actual_data = swh_filter_renderer.filter_by_fields(input_data) # then self.assertEquals(actual_data, input_data) @patch('swh.web.ui.renderers.utils') @patch('swh.web.ui.renderers.request') @istest def swh_filter_renderer_do_filter(self, mock_request, mock_utils): # given mock_request.args = {'fields': 'a,c'} mock_utils.filter_field_keys.return_value = {'a': 'some-data'} swh_filter_renderer = renderers.SWHFilterEnricher() input_data = {'a': 'some-data', 'b': 'some-other-data'} # when actual_data = swh_filter_renderer.filter_by_fields(input_data) # then self.assertEquals(actual_data, {'a': 'some-data'}) mock_utils.filter_field_keys.assert_called_once_with(input_data, {'a', 'c'}) @patch('swh.web.ui.renderers.request') @istest def yaml_renderer_without_filter(self, mock_request): # given mock_request.args = {} yaml_renderer = renderers.YAMLRenderer() input_data = {'target': 'sha1-dir', 'type': 'dir', 'dir-id': 'dir-id-sha1-git'} expected_data = input_data # when actual_data = yaml_renderer.render(input_data, 'application/yaml') # then self.assertEqual(yaml.load(actual_data), expected_data) @patch('swh.web.ui.renderers.request') @istest def yaml_renderer(self, mock_request): # given mock_request.args = {'fields': 'type,target'} yaml_renderer = renderers.YAMLRenderer() input_data = {'target': 'sha1-dir', 'type': 'dir', 'dir-id': 'dir-id-sha1-git'} expected_data = {'target': 'sha1-dir', 'type': 'dir'} # when actual_data = yaml_renderer.render(input_data, 'application/yaml') # then self.assertEqual(yaml.load(actual_data), expected_data) @patch('swh.web.ui.renderers.request') @istest def json_renderer_basic(self, mock_request): # given mock_request.args = {} json_renderer = renderers.SWHJSONRenderer() input_data = {'target': 'sha1-dir', 'type': 'dir', 'dir-id': 'dir-id-sha1-git'} expected_data = input_data # when actual_data = json_renderer.render(input_data, MediaType( 'application/json')) # then self.assertEqual(json.loads(actual_data), expected_data) @patch('swh.web.ui.renderers.request') @istest def json_renderer_basic_with_filter(self, mock_request): # given mock_request.args = {'fields': 'target'} json_renderer = renderers.SWHJSONRenderer() input_data = {'target': 'sha1-dir', 'type': 'dir', 'dir-id': 'dir-id-sha1-git'} expected_data = {'target': 'sha1-dir'} # when actual_data = json_renderer.render(input_data, MediaType( 'application/json')) # then self.assertEqual(json.loads(actual_data), expected_data) @patch('swh.web.ui.renderers.request') @istest def json_renderer_basic_with_filter_and_jsonp(self, mock_request): # given mock_request.args = {'fields': 'target', 'callback': 'jsonpfn'} json_renderer = renderers.SWHJSONRenderer() input_data = {'target': 'sha1-dir', 'type': 'dir', 'dir-id': 'dir-id-sha1-git'} # when actual_data = json_renderer.render(input_data, MediaType( 'application/json')) # then self.assertEqual(actual_data, 'jsonpfn({"target": "sha1-dir"})') @patch('swh.web.ui.renderers.request') @istest def jsonp_enricher_basic_with_filter_and_jsonp(self, mock_request): # given mock_request.args = {'callback': 'jsonpfn'} jsonp_enricher = renderers.JSONPEnricher() # when actual_output = jsonp_enricher.enrich_with_jsonp({'output': 'test'}) # then self.assertEqual(actual_output, "jsonpfn({'output': 'test'})") @patch('swh.web.ui.renderers.request') @istest def jsonp_enricher_do_nothing(self, mock_request): # given mock_request.args = {} jsonp_enricher = renderers.JSONPEnricher() # when actual_output = jsonp_enricher.enrich_with_jsonp({'output': 'test'}) # then self.assertEqual(actual_output, {'output': 'test'}) @istest def urlize_api_links(self): # update api link with html links content with links content = '{"url": "/api/1/abc/"}' expected_content = '{"url": "/api/1/abc/"}' self.assertEquals(renderers.urlize_api_links(content), expected_content) # update /browse link with html links content with links content = '{"url": "/browse/def/"}' expected_content = '{"url": "' \ '/browse/def/"}' self.assertEquals(renderers.urlize_api_links(content), expected_content) # will do nothing since it's not an api url other_content = '{"url": "/something/api/1/other"}' self.assertEquals(renderers.urlize_api_links(other_content), other_content) @istest def revision_id_from_url(self): url = ('/browse/revision/9ba4bcb645898d562498ea66a0df958ef0e7a68c/' 'prev/9ba4bcb645898d562498ea66a0df958ef0e7aaaa/') expected_id = '9ba4bcb645898d562498ea66a0df958ef0e7a68c' self.assertEqual(renderers.revision_id_from_url(url), expected_id) @istest def safe_docstring_display(self): # update api link with html links content with links - docstring = """

Show all revisions (~git log) starting from -sha1_git. - The first element returned is the given sha1_git.

-

Args: - sha1_git: the revision's hash

-

Returns: - Information on the revision if found.

-

Raises: - BadInputExc in case of unknown algo_hash or bad hash - NotFoundExc if the revision is not found.

-

Example: - blah

""" - expected_docstring = """

Show all revisions (~git log) starting from -sha1_git. - The first element returned is the given sha1_git.

-

Args:

   - sha1_git: the revision's hash

-

Returns:

   - Information on the revision if found.

-

Raises:

   - BadInputExc in case of unknown algo_hash or bad hash - NotFoundExc if the revision is not found.

-

Example:

   - blah

""" + docstring = """This is my list header: + + - Here is item 1, with a continuation + line right here + - Here is item 2 + + Here is something that is not part of the list""" + + expected_docstring = """

This is my list header:

+ +

Here is something that is not part of the list

+""" self.assertEquals(renderers.safe_docstring_display(docstring), expected_docstring) diff --git a/swh/web/ui/views/api.py b/swh/web/ui/views/api.py index 0c67a4e4..008b0ec2 100644 --- a/swh/web/ui/views/api.py +++ b/swh/web/ui/views/api.py @@ -1,735 +1,739 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from types import GeneratorType from flask import request, url_for from swh.web.ui import service, utils, apidoc as doc from swh.web.ui.exc import NotFoundExc from swh.web.ui.main import app @app.route('/api/1/stat/counters/') @doc.route('/api/1/stat/counters/', noargs=True) @doc.returns(rettype=doc.rettypes.dict, retdoc="A dictionary of SWH's most important statistics") def api_stats(): """Return statistics on SWH storage. """ return service.stat_counters() @app.route('/api/1/stat/visits//') @doc.route('/api/1/stat/visits/') @doc.arg('origin_id', default=1, argtype=doc.argtypes.int, argdoc='The requested SWH origin identifier') @doc.returns(rettype=doc.rettypes.list, retdoc="""All instances of visits of the origin pointed by origin_id as POSIX time since epoch""") def api_origin_visits(origin_id): """Return a list of visit dates as POSIX timestamps for the given revision. """ date_gen = (item['date'] for item in service.stat_origin_visits(origin_id)) return sorted(date_gen) @app.route('/api/1/search/', methods=['POST']) @app.route('/api/1/search//') @doc.route('/api/1/search/') @doc.arg('q', default='sha1:adc83b19e793491b1c6ea0fd8b46cd9f32e592fc', argtype=doc.argtypes.algo_and_hash, argdoc="""An algo_hash:hash string, where algo_hash is one of sha1, sha1_git or sha256 and hash is the hash to search for in SWH""") @doc.raises(exc=doc.excs.badinput, doc='Raised if q is not well formed') @doc.returns(rettype=doc.rettypes.dict, retdoc="""A dict with keys: - * search_res: a list of dicts corresponding to queried content - with key 'found' to True if found, 'False' if not - * search_stats: a dict containing number of files searched and - percentage of files found + + - search_res: a list of dicts corresponding to queried content + with key 'found' to True if found, 'False' if not + - search_stats: a dict containing number of files searched and + percentage of files found """) def api_search(q=None): """Search a content per hash. - This may take the form of a GET request with a single checksum, or a POST - request with many hashes, with the request body containing identifiers - (typically filenames) as keys and corresponding hashes as values. + This may take the form of: + + - a GET request with a single checksum + - a POST request with many hashes, with the request body containing + identifiers (typically filenames) as keys and corresponding hashes as + values. """ response = {'search_res': None, 'search_stats': None} search_stats = {'nbfiles': 0, 'pct': 0} search_res = None # Single hash request route if q: r = service.search_hash(q) search_res = [{'filename': None, 'sha1': q, 'found': r['found']}] search_stats['nbfiles'] = 1 search_stats['pct'] = 100 if r['found'] else 0 # Post form submission with many hash requests elif request.method == 'POST': data = request.form queries = [] # Remove potential inputs with no associated value for k, v in data.items(): if v is not None: if k == 'q' and len(v) > 0: queries.append({'filename': None, 'sha1': v}) elif v != '': queries.append({'filename': k, 'sha1': v}) if len(queries) > 0: lookup = service.lookup_multiple_hashes(queries) result = [] for el in lookup: result.append({'filename': el['filename'], 'sha1': el['sha1'], 'found': el['found']}) search_res = result nbfound = len([x for x in lookup if x['found']]) search_stats['nbfiles'] = len(queries) search_stats['pct'] = (nbfound / len(queries))*100 response['search_res'] = search_res response['search_stats'] = search_stats return response def _api_lookup(criteria, lookup_fn, error_msg_if_not_found, enrich_fn=lambda x: x, *args): """Capture a redundant behavior of: - looking up the backend with a criteria (be it an identifier or checksum) passed to the function lookup_fn - if nothing is found, raise an NotFoundExc exception with error message error_msg_if_not_found. - Otherwise if something is returned: - either as list, map or generator, map the enrich_fn function to it and return the resulting data structure as list. - either as dict and pass to enrich_fn and return the dict enriched. Args: - criteria: discriminating criteria to lookup - lookup_fn: function expects one criteria and optional supplementary *args. - error_msg_if_not_found: if nothing matching the criteria is found, raise NotFoundExc with this error message. - enrich_fn: Function to use to enrich the result returned by lookup_fn. Default to the identity function if not provided. - *args: supplementary arguments to pass to lookup_fn. Raises: NotFoundExp or whatever `lookup_fn` raises. """ res = lookup_fn(criteria, *args) if not res: raise NotFoundExc(error_msg_if_not_found) if isinstance(res, (map, list, GeneratorType)): enriched_data = [] for e in res: enriched_data.append(enrich_fn(e)) return enriched_data return enrich_fn(res) @app.route('/api/1/origin//') @doc.route('/api/1/origin/') @doc.arg('origin_id', default=1, argtype=doc.argtypes.int, argdoc="The origin's SWH origin_id.") @doc.raises(exc=doc.excs.notfound, doc='Raised if origin_id does not correspond to an origin in SWH') @doc.returns(rettype=doc.rettypes.dict, retdoc='The metadata of the origin identified by origin_id') def api_origin(origin_id): """Return information about origin with id origin_id. """ return _api_lookup( origin_id, lookup_fn=service.lookup_origin, error_msg_if_not_found='Origin with id %s not found.' % origin_id) @app.route('/api/1/person//') @doc.route('/api/1/person/') @doc.arg('person_id', default=1, argtype=doc.argtypes.int, argdoc="The person's SWH identifier") @doc.raises(exc=doc.excs.notfound, doc='Raised if person_id does not correspond to an origin in SWH') @doc.returns(rettype=doc.rettypes.dict, retdoc='The metadata of the person identified by person_id') def api_person(person_id): """Return information about person with identifier person_id. """ return _api_lookup( person_id, lookup_fn=service.lookup_person, error_msg_if_not_found='Person with id %s not found.' % person_id) @app.route('/api/1/release//') @doc.route('/api/1/release/') @doc.arg('sha1_git', default='8b137891791fe96927ad78e64b0aad7bded08bdc', argtype=doc.argtypes.sha1_git, argdoc="The release's sha1_git identifier") @doc.raises(exc=doc.excs.badinput, doc='Raised if the argument is not a sha1') @doc.raises(exc=doc.excs.notfound, doc='Raised if sha1_git does not correspond to a release in SWH') @doc.returns(rettype=doc.rettypes.dict, retdoc='The metadata of the release identified by sha1_git') def api_release(sha1_git): """Return information about release with id sha1_git. """ error_msg = 'Release with sha1_git %s not found.' % sha1_git return _api_lookup( sha1_git, lookup_fn=service.lookup_release, error_msg_if_not_found=error_msg, enrich_fn=utils.enrich_release) def _revision_directory_by(revision, path, request_path, limit=100, with_data=False): """Compute the revision matching criterion's directory or content data. Args: revision: dictionary of criterions representing a revision to lookup path: directory's path to lookup request_path: request path which holds the original context to limit: optional query parameter to limit the revisions log (default to 100). For now, note that this limit could impede the transitivity conclusion about sha1_git not being an ancestor of with_data: indicate to retrieve the content's raw data if path resolves to a content. """ def enrich_directory_local(dir, context_url=request_path): return utils.enrich_directory(dir, context_url) rev_id, result = service.lookup_directory_through_revision( revision, path, limit=limit, with_data=with_data) content = result['content'] if result['type'] == 'dir': # dir_entries result['content'] = list(map(enrich_directory_local, content)) else: # content result['content'] = utils.enrich_content(content) return result @app.route('/api/1/revision' '/origin/' '/directory/') @app.route('/api/1/revision' '/origin/' '/directory//') @app.route('/api/1/revision' '/origin/' '/branch/' '/directory/') @app.route('/api/1/revision' '/origin/' '/branch/' '/directory//') @app.route('/api/1/revision' '/origin/' '/branch/' '/ts/' '/directory/') @app.route('/api/1/revision' '/origin/' '/branch/' '/ts/' '/directory//') @doc.route('/api/1/revision/origin/directory/') @doc.arg('origin_id', default=1, argtype=doc.argtypes.int, argdoc="The revision's origin's SWH identifier") @doc.arg('branch_name', default='refs/heads/master', argtype=doc.argtypes.path, argdoc="""The optional branch for the given origin (default to master""") @doc.arg('ts', default='2000-01-17T11:23:54+00:00', argtype=doc.argtypes.ts, argdoc="""Optional timestamp (default to the nearest time crawl of timestamp)""") @doc.arg('path', default='.', argtype=doc.argtypes.path, argdoc='The path to the directory or file to display') @doc.raises(exc=doc.excs.notfound, doc="""Raised if a revision matching the passed criteria was not found""") @doc.returns(rettype=doc.rettypes.dict, retdoc="""The metadata of the revision corresponding to the passed criteria""") def api_directory_through_revision_origin(origin_id, branch_name="refs/heads/master", ts=None, path=None, with_data=False): """Display directory or content information through a revision identified by origin/branch/timestamp. """ if ts: ts = utils.parse_timestamp(ts) return _revision_directory_by( { 'origin_id': origin_id, 'branch_name': branch_name, 'ts': ts }, path, request.path, with_data=with_data) @app.route('/api/1/revision' '/origin//') @app.route('/api/1/revision' '/origin/' '/branch//') @app.route('/api/1/revision' '/origin/' '/branch/' '/ts//') @app.route('/api/1/revision' '/origin/' '/ts//') @doc.route('/api/1/revision/origin/') @doc.arg('origin_id', default=1, argtype=doc.argtypes.int, argdoc="The queried revision's origin identifier in SWH") @doc.arg('branch_name', default='refs/heads/master', argtype=doc.argtypes.path, argdoc="""The optional branch for the given origin (default to master)""") @doc.arg('ts', default='2000-01-17T11:23:54+00:00', argtype=doc.argtypes.ts, argdoc="The time at which the queried revision should be constrained") @doc.raises(exc=doc.excs.notfound, doc="""Raised if a revision matching given criteria was not found in SWH""") @doc.returns(rettype=doc.rettypes.dict, retdoc="""The metadata of the revision identified by the given criteria""") def api_revision_with_origin(origin_id, branch_name="refs/heads/master", ts=None): """Display revision information through its identification by origin/branch/timestamp. """ if ts: ts = utils.parse_timestamp(ts) return _api_lookup( origin_id, service.lookup_revision_by, 'Revision with (origin_id: %s, branch_name: %s' ', ts: %s) not found.' % (origin_id, branch_name, ts), utils.enrich_revision, branch_name, ts) @app.route('/api/1/revision//') @app.route('/api/1/revision//prev//') @doc.route('/api/1/revision/') @doc.arg('sha1_git', default='ec72c666fb345ea5f21359b7bc063710ce558e39', argtype=doc.argtypes.sha1_git, argdoc="The revision's sha1_git identifier") @doc.arg('context', default='6adc4a22f20bbf3bbc754f1ec8c82be5dfb5c71a', argtype=doc.argtypes.path, argdoc='The navigation breadcrumbs -- use at your own risk') @doc.raises(exc=doc.excs.badinput, doc='Raised if sha1_git is not well formed') @doc.raises(exc=doc.excs.notfound, doc='Raised if a revision matching sha1_git was not found in SWH') @doc.returns(rettype=doc.rettypes.dict, retdoc='The metadata of the revision identified by sha1_git') def api_revision(sha1_git, context=None): """Return information about revision with id sha1_git. """ def _enrich_revision(revision, context=context): return utils.enrich_revision(revision, context) return _api_lookup( sha1_git, service.lookup_revision, 'Revision with sha1_git %s not found.' % sha1_git, _enrich_revision) @app.route('/api/1/revision//raw/') @doc.route('/api/1/revision/raw/') @doc.arg('sha1_git', default='ec72c666fb345ea5f21359b7bc063710ce558e39', argtype=doc.argtypes.sha1_git, argdoc="The queried revision's sha1_git identifier") @doc.raises(exc=doc.excs.badinput, doc='Raised if sha1_git is not well formed') @doc.raises(exc=doc.excs.notfound, doc='Raised if a revision matching sha1_git was not found in SWH') @doc.returns(rettype=doc.argtypes.octet_stream, retdoc="""The message of the revision identified by sha1_git as a downloadable octet stream""") def api_revision_raw_message(sha1_git): """Return the raw data of the message of revision identified by sha1_git """ raw = service.lookup_revision_message(sha1_git) return app.response_class(raw['message'], headers={'Content-disposition': 'attachment;' 'filename=rev_%s_raw' % sha1_git}, mimetype='application/octet-stream') @app.route('/api/1/revision//directory/') @app.route('/api/1/revision//directory//') @doc.route('/api/1/revision/directory/') @doc.arg('sha1_git', default='ec72c666fb345ea5f21359b7bc063710ce558e39', argtype=doc.argtypes.sha1_git, argdoc="The revision's sha1_git identifier.") @doc.arg('dir_path', default='.', argtype=doc.argtypes.path, argdoc='The path from the top level directory') @doc.raises(exc=doc.excs.badinput, doc='Raised if sha1_git is not well formed') @doc.raises(exc=doc.excs.notfound, doc="""Raised if a revision matching sha1_git was not found in SWH , or if the path specified does not exist""") @doc.returns(rettype=doc.rettypes.dict, retdoc="""The metadata of the directory pointed by revision id sha1-git and dir_path""") def api_revision_directory(sha1_git, dir_path=None, with_data=False): """Return information on directory pointed by revision with sha1_git. If dir_path is not provided, display top level directory. Otherwise, display the directory pointed by dir_path (if it exists). """ return _revision_directory_by( { 'sha1_git': sha1_git }, dir_path, request.path, with_data=with_data) @app.route('/api/1/revision//log/') @app.route('/api/1/revision//prev//log/') @doc.route('/api/1/revision/log/') @doc.arg('sha1_git', default='ec72c666fb345ea5f21359b7bc063710ce558e39', argtype=doc.argtypes.sha1_git, argdoc='The sha1_git of the revision queried') @doc.arg('prev_sha1s', default='6adc4a22f20bbf3bbc754f1ec8c82be5dfb5c71a', argtype=doc.argtypes.path, argdoc='The navigation breadcrumbs -- use at your own risk!') @doc.raises(exc=doc.excs.badinput, doc='Raised if sha1_git or prev_sha1s is not well formed') @doc.raises(exc=doc.excs.notfound, doc='Raised if a revision matching sha1_git was not found in SWH') @doc.returns(rettype=doc.rettypes.dict, retdoc="""The log data starting at the revision identified by sha1_git, completed with the navigation breadcrumbs, if any""") def api_revision_log(sha1_git, prev_sha1s=None): """Show all revisions (~git log) starting from sha1_git. The first element returned is the given sha1_git, or the first breadcrumb, if any. """ limit = app.config['conf']['max_log_revs'] response = {'revisions': None, 'next_revs_url': None} revisions = None next_revs_url = None def lookup_revision_log_with_limit(s, limit=limit+1): return service.lookup_revision_log(s, limit) error_msg = 'Revision with sha1_git %s not found.' % sha1_git rev_get = _api_lookup(sha1_git, lookup_fn=lookup_revision_log_with_limit, error_msg_if_not_found=error_msg, enrich_fn=utils.enrich_revision) if len(rev_get) == limit+1: rev_backward = rev_get[:-1] next_revs_url = url_for('api_revision_log', sha1_git=rev_get[-1]['id']) else: rev_backward = rev_get if not prev_sha1s: # no nav breadcrumbs, so we're done revisions = rev_backward else: rev_forward_ids = prev_sha1s.split('/') rev_forward = _api_lookup(rev_forward_ids, lookup_fn=service.lookup_revision_multiple, error_msg_if_not_found=error_msg, enrich_fn=utils.enrich_revision) revisions = rev_forward + rev_backward response['revisions'] = revisions response['next_revs_url'] = next_revs_url return response @app.route('/api/1/revision' '/origin//log/') @app.route('/api/1/revision' '/origin/' '/branch//log/') @app.route('/api/1/revision' '/origin/' '/branch/' '/ts//log/') @app.route('/api/1/revision' '/origin/' '/ts//log/') @doc.route('/api/1/revision/origin/log/') @doc.arg('origin_id', default=1, argtype=doc.argtypes.int, argdoc="The revision's SWH origin identifier") @doc.arg('branch_name', default='refs/heads/master', argtype=doc.argtypes.path, argdoc="The revision's branch name within the origin specified") @doc.arg('ts', default='2000-01-17T11:23:54+00:00', argtype=doc.argtypes.ts, argdoc="""A time or timestamp string to parse""") @doc.raises(exc=doc.excs.notfound, doc="""Raised if a revision matching the given criteria was not found in SWH""") @doc.returns(rettype=doc.rettypes.dict, retdoc="""The metadata of the revision log starting at the revision matching the given criteria.""") def api_revision_log_by(origin_id, branch_name='refs/heads/master', ts=None): """Show all revisions (~git log) starting from the revision described by its origin_id, optional branch name and timestamp. The first element returned is the described revision. """ limit = app.config['conf']['max_log_revs'] response = {'revisions': None, 'next_revs_url': None} next_revs_url = None if ts: ts = utils.parse_timestamp(ts) def lookup_revision_log_by_with_limit(o_id, br, ts, limit=limit+1): return service.lookup_revision_log_by(o_id, br, ts, limit) error_msg = 'No revision matching origin %s ' % origin_id error_msg += ', branch name %s' % branch_name error_msg += (' and time stamp %s.' % ts) if ts else '.' rev_get = _api_lookup(origin_id, lookup_revision_log_by_with_limit, error_msg, utils.enrich_revision, branch_name, ts) if len(rev_get) == limit+1: revisions = rev_get[:-1] next_revs_url = url_for('api_revision_log', sha1_git=rev_get[-1]['id']) else: revisions = rev_get response['revisions'] = revisions response['next_revs_url'] = next_revs_url return response @app.route('/api/1/directory//') @app.route('/api/1/directory///') @doc.route('/api/1/directory/') @doc.arg('sha1_git', default='adc83b19e793491b1c6ea0fd8b46cd9f32e592fc', argtype=doc.argtypes.sha1_git, argdoc="The queried directory's corresponding sha1_git hash") @doc.arg('path', default='.', argtype=doc.argtypes.path, argdoc="A path relative to the queried directory's top level") @doc.raises(exc=doc.excs.badinput, doc='Raised if sha1_git is not well formed') @doc.raises(exc=doc.excs.notfound, doc='Raised if a directory matching sha1_git was not found in SWH') @doc.returns(rettype=doc.rettypes.dict, retdoc="""The metadata and contents of the release identified by sha1_git""") def api_directory(sha1_git, path=None): """Return information about release with id sha1_git. """ if path: error_msg_path = ('Entry with path %s relative to directory ' 'with sha1_git %s not found.') % (path, sha1_git) return _api_lookup( sha1_git, service.lookup_directory_with_path, error_msg_path, utils.enrich_directory, path) else: error_msg_nopath = 'Directory with sha1_git %s not found.' % sha1_git return _api_lookup( sha1_git, service.lookup_directory, error_msg_nopath, utils.enrich_directory) # @app.route('/api/1/browse/') # @app.route('/api/1/browse//') def api_content_checksum_to_origin(q): """Return content information up to one of its origin if the content is found. Args: q is of the form algo_hash:hash with algo_hash in (sha1, sha1_git, sha256). Returns: Information on one possible origin for such content. Raises: BadInputExc in case of unknown algo_hash or bad hash. NotFoundExc if the content is not found. Example: GET /api/1/browse/sha1_git:88b9b366facda0b5ff8d8640ee9279bed346f242 """ found = service.lookup_hash(q)['found'] if not found: raise NotFoundExc('Content with %s not found.' % q) return service.lookup_hash_origin(q) @app.route('/api/1/content//raw/') @doc.route('/api/1/content/raw/') @doc.arg('q', default='adc83b19e793491b1c6ea0fd8b46cd9f32e592fc', argtype=doc.argtypes.algo_and_hash, argdoc="""An algo_hash:hash string, where algo_hash is one of sha1, sha1_git or sha256 and hash is the hash to search for in SWH. Defaults to sha1 in the case of a missing algo_hash """) @doc.raises(exc=doc.excs.badinput, doc='Raised if q is not well formed') @doc.raises(exc=doc.excs.notfound, doc='Raised if a content matching q was not found in SWH') @doc.returns(rettype='octet stream', retdoc='The raw content data as an octet stream') def api_content_raw(q): """Return content's raw data if content is found. """ def generate(content): yield content['data'] content = service.lookup_content_raw(q) if not content: raise NotFoundExc('Content with %s not found.' % q) return app.response_class(generate(content), headers={'Content-disposition': 'attachment;' 'filename=content_%s_raw' % q}, mimetype='application/octet-stream') @app.route('/api/1/content//') @doc.route('/api/1/content/') @doc.arg('q', default='adc83b19e793491b1c6ea0fd8b46cd9f32e592fc', argtype=doc.argtypes.algo_and_hash, argdoc="""An algo_hash:hash string, where algo_hash is one of sha1, sha1_git or sha256 and hash is the hash to search for in SWH. Defaults to sha1 in the case of a missing algo_hash """) @doc.raises(exc=doc.excs.badinput, doc='Raised if q is not well formed') @doc.raises(exc=doc.excs.notfound, doc='Raised if a content matching q was not found in SWH') @doc.returns(rettype=doc.rettypes.dict, retdoc="""The metadata of the content identified by q. If content decoding was successful, it also returns the data""") def api_content_metadata(q): """Return content information if content is found. """ return _api_lookup( q, lookup_fn=service.lookup_content, error_msg_if_not_found='Content with %s not found.' % q, enrich_fn=utils.enrich_content) @app.route('/api/1/entity//') @doc.route('/api/1/entity/') @doc.arg('uuid', default='5f4d4c51-498a-4e28-88b3-b3e4e8396cba', argtype=doc.argtypes.uuid, argdoc="The entity's uuid identifier") @doc.raises(exc=doc.excs.badinput, doc='Raised if uuid is not well formed') @doc.raises(exc=doc.excs.notfound, doc='Raised if an entity matching uuid was not found in SWH') @doc.returns(rettype=doc.rettypes.dict, retdoc='The metadata of the entity identified by uuid') def api_entity_by_uuid(uuid): """Return content information if content is found. """ return _api_lookup( uuid, lookup_fn=service.lookup_entity_by_uuid, error_msg_if_not_found="Entity with uuid '%s' not found." % uuid, enrich_fn=utils.enrich_entity)