diff --git a/README-uri-scheme.md b/README-uri-scheme.md index 81623017..8b0f4ea3 100644 --- a/README-uri-scheme.md +++ b/README-uri-scheme.md @@ -1,248 +1,252 @@ URI scheme ========== Browsing namespace ------------------ ### Global The api /api/1 is partially browsable on defined endpoints. Also, you can specify specify 'Accept' header using your favorite client side to see the answer being transformed accordingly. Support of the following 'Accept' header: - application/json - application/yaml - text/html To be anchored where browsing starts (e.g., at /api/1) +* /api/ and /api/1/ + +List endpoint methods as per the client's 'Accept' header request. + * /revision/: show commit information $ curl -H 'Accept: application/json' http://localhost:6543/api/1/revision/18d8be353ed3480476f032475e7c233eff7371d5 { "author_email": "robot@softwareheritage.org", "author_name": "Software Heritage", "committer_date": "Mon, 17 Jan 2000 10:23:54 GMT", "committer_date_offset": 0, "committer_email": "robot@softwareheritage.org", "committer_name": "Software Heritage", "date": "Mon, 17 Jan 2000 10:23:54 GMT", "date_offset": 0, "directory": "7834ef7e7c357ce2af928115c6c6a42b7e2a44e6", "id": "18d8be353ed3480476f032475e7c233eff7371d5", "message": "synthetic revision message", "metadata": { "original_artifact": [ { "archive_type": "tar", "name": "webbase-5.7.0.tar.gz", "sha1": "147f73f369733d088b7a6fa9c4e0273dcd3c7ccd", "sha1_git": "6a15ea8b881069adedf11feceec35588f2cfe8f1", "sha256": "401d0df797110bea805d358b85bcc1ced29549d3d73f309d36484e7edf7bb912" } ] }, "parents": [ null ], "synthetic": true, "type": "tar" } * /directory/: show directory information (including ls) curl -X GET http://localhost:6543/api/1/directory/3126f46e2f7dc752227131a2a658265e58f53e38?recursive=True [ { "dir_id": "3126f46e2f7dc752227131a2a658265e58f53e38", "name": "Makefile.am", "perms": 100644, "sha1": "b0283d8126f975e7b4a4348d13b07ddebe2cf8bf", "sha1_git": "e0522786777256d57c5210219bcbe8dacdad273d", "sha256": "897f3189dcfba96281b2190325c54afc74a42e2419c053baadfadc14386935ee", "status": "visible", "target": "e0522786777256d57c5210219bcbe8dacdad273d", "type": "file" }, { "dir_id": "3126f46e2f7dc752227131a2a658265e58f53e38", "name": "Makefile.in", "perms": 100644, "sha1": "81f5757b9451811cfb3ef84612e45a973c70b4e6", "sha1_git": "3b948d966fd8e99f93670025f63a550168d57d71", "sha256": "f5acd84a40f05d997a36b8846c4872a92ee57083abb77c82e05e9763c8edb59a", "status": "visible", "target": "3b948d966fd8e99f93670025f63a550168d57d71", "type": "file" }, ... snip ... { "dir_id": "3126f46e2f7dc752227131a2a658265e58f53e38", "name": "webtools.h", "perms": 100644, "sha1": "4b4c942ddd490ec1e312074ddfac352097886c02", "sha1_git": "e6fb8969d00e23dd152df5e7fb167118eab67342", "sha256": "95ffe6c0108f6ec48ccb0c93e966b54f1494f5cc353b066644c11fa47766620f", "status": "visible", "target": "e6fb8969d00e23dd152df5e7fb167118eab67342", "type": "file" }, { "dir_id": "3126f46e2f7dc752227131a2a658265e58f53e38", "name": "ylwrap", "perms": 100644, "sha1": "9073938df9ae47d585bfdf176bfff45d06f3e13e", "sha1_git": "13fc38d75f2a47bc55e90ad5bf8d8a0184b14878", "sha256": "184eb644e51154c79b42df70c22955b818d057491f84ca0e579e4f9e48a60d7b", "status": "visible", "target": "13fc38d75f2a47bc55e90ad5bf8d8a0184b14878", "type": "file" } ] * /directory//path/to/file-or-dir: ditto, but for dir pointed by path - note: this is the same as /dir/, where is the sha1_git ID of the dir pointed by path * /content/[:]: show content information - content is specified by HASH, according to HASH_ALGO, where HASH_ALGO is one of: sha1, sha1_git, sha256. This means that several different URLs (at least one per HASH_ALGO) will point to the same content - HASH_ALGO defaults to "sha1" (?) curl -X GET http://localhost:6543/api/1/content/sha1:486b486d2a4998929c68265fa85ab2326db5528a { "data": "/api/1/content/486b486d2a4998929c68265fa85ab2326db5528a/raw", "sha1": "486b486d2a4998929c68265fa85ab2326db5528a" } curl -X GET http://localhost:6543/api/1/content/sha1:4a1b6d7dd0a923ed90156c4e2f5db030095d8e08/ {"error": "Content with sha1:4a1b6d7dd0a923ed90156c4e2f5db030095d8e08 not found."} * /content/[/raw curl -H 'Accept: text/plain' http://localhost:6543/api/1/content/sha1:486b486d2a4998929c68265fa85ab2326db5528a/raw The GNU cfs-el web homepage is at @uref{http://www.gnu.org/software/cfs-el/cfs-el.html}. You can find the latest distribution of GNU cfs-el at @uref{ftp://ftp.gnu.org/gnu/} or at any of its mirrors. * /release/: show release information Sample: $ curl -X GET http://localhost:6543/api/1/release/4a1b6d7dd0a923ed90156c4e2f5db030095d8e08 { "author_name": "Software Heritage", "author_email": "robot@softwareheritage.org", "comment": "synthetic release message", "date": "Sat, 04 Mar 2000 07:50:35 GMT", "date_offset": 0, "id": "4a1b6d7dd0a923ed90156c4e2f5db030095d8e08", "name": "4.0.6", "revision": "5c7814ce9978d4e16f3858925b5cea611e500eec", "synthetic": true }% * /person/: show person information curl http://localhost:6543/api/1/person/1 { "email": "robot@softwareheritage.org", "id": 1, "name": "Software Heritage" } curl http://localhost:6543/api/1/person/2 {"error": "Person with id 2 not found."} * /origin/: show origin information Sample: $ curl -X GET http://localhost:6543/api/1/origin/1 { "id": 1, "lister": null, "project": null, "type": "ftp", "url": "rsync://ftp.gnu.org/old-gnu/solfege" }% * /project/: show project information * /organization/: show organization information * /browse/ Return content information up to one of its origin if the content is found. curl http://localhost:6543/api/1/browse/sha1:2e98ab73456aad8dfc6cc50d562ee1b80d201753 { "path": "republique.py", "origin_url": "file:///dev/null", "origin_type": "git", "revision": "8f8640a1c024c2ef85fa8e8d9297ea289134472d", "branch": "refs/remotes/origin/master" } ### Occurrence Origin/Branch do not contain `|` so it is used as a terminator. Origin is . Timestamp is one of: latest or an ISO8601 date (TODO: decide the time matching policy). * /directory//|/|/path/to/file-or-dir - Same as /directory/ but looking up sha1 git using origin and branch at a given timestamp * /revision//|/ - Same as /revision/ but looking up sha1 git using origin and branch at a given timestamp * /revision//| - Show all branches of origin at a given timestamp * /revision//|/| - Show all revisions (~git log) of origin and branch at a given timestamp ### Upload and search * /1/api/uploadnsearch/ Post a file's content to api. Api computes the sha1 hash and checks in the storage if such sha1 exists. Json answer: {'sha1': hexadecimal sha1, 'found': true or false} Sample: $ curl -X POST -F filename=@/path/to/file http://localhost:6543/api/1/uploadnsearch { "found": false, "sha1": "e95097ad2d607b4c89c1ce7ca1fef2a1e4450558" }% Search namespace ---------------- diff --git a/swh/web/ui/renderers.py b/swh/web/ui/renderers.py index 6532cbe9..c9d414d7 100644 --- a/swh/web/ui/renderers.py +++ b/swh/web/ui/renderers.py @@ -1,95 +1,109 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import yaml from flask import make_response, request from flask.ext.api import renderers, parsers from flask_api.mediatypes import MediaType +from swh.web.ui import utils + + +class SWHFilterRenderer(): + """Base renderer for swh's common behavior. + + """ + def filter_by_fields(self, data): + fields = request.args.get('fields') + if fields: + data = utils.filter_field_keys(data, fields) + + return data class PlainRenderer(renderers.BaseRenderer): """Renderer for plain/text, do nothing but send the data as is. """ media_type = 'text/plain' def render(self, data, media_type, **options): return data -class YAMLRenderer(renderers.BaseRenderer): +class YAMLRenderer(renderers.BaseRenderer, SWHFilterRenderer): """Renderer for application/yaml. Orchestrate from python data structure to yaml. """ media_type = 'application/yaml' def render(self, data, media_type, **options): + data = self.filter_by_fields(data) return yaml.dump(data, encoding=self.charset) -class JSONPRenderer(renderers.JSONRenderer): +class JSONPRenderer(renderers.JSONRenderer, SWHFilterRenderer): """Renderer for application/json. Serializes in json the data and returns it. Also deals with jsonp. If callback is found in request parameter, wrap the result as a function with name the value of the parameter query 'callback'. """ def render(self, data, media_type, **options): - # Requested indentation may be set in the Accept header. + data = self.filter_by_fields(data) res = super().render(data, media_type, **options) jsonp = request.args.get('callback') if jsonp: return '%s(%s)' % (jsonp, res) return res RENDERERS = [ 'swh.web.ui.renderers.JSONPRenderer', 'flask.ext.api.renderers.BrowsableAPIRenderer', 'flask.ext.api.parsers.URLEncodedParser', 'swh.web.ui.renderers.YAMLRenderer', 'swh.web.ui.renderers.PlainRenderer', ] RENDERERS_INSTANCE = [ JSONPRenderer(), renderers.BrowsableAPIRenderer(), parsers.URLEncodedParser(), YAMLRenderer(), ] RENDERERS_BY_TYPE = { r.media_type: r for r in RENDERERS_INSTANCE } def error_response(default_error_msg, error_code, error): """Private function to create a custom error response. """ # if nothing is requested by client, use json default_application_type = 'application/json' accept_type = request.headers.get('Accept', default_application_type) renderer = RENDERERS_BY_TYPE.get( accept_type, RENDERERS_BY_TYPE[default_application_type]) # for edge cases, use the elected renderer's media type accept_type = renderer.media_type response = make_response(default_error_msg, error_code) response.headers['Content-Type'] = accept_type response.data = renderer.render({"error": str(error)}, media_type=MediaType(accept_type), status=error_code, headers={'Content-Type': accept_type}) return response diff --git a/swh/web/ui/tests/test_renderers.py b/swh/web/ui/tests/test_renderers.py new file mode 100644 index 00000000..2d1c7cd0 --- /dev/null +++ b/swh/web/ui/tests/test_renderers.py @@ -0,0 +1,163 @@ +# Copyright (C) 2015 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import json +import unittest +import yaml + +from flask_api.mediatypes import MediaType +from nose.tools import istest +from unittest.mock import patch + +from swh.web.ui import renderers + + +class RendererTestCase(unittest.TestCase): + + @patch('swh.web.ui.renderers.request') + @istest + def SWHFilterRenderer_do_nothing(self, mock_request): + # given + mock_request.args = {} + + swhFilterRenderer = renderers.SWHFilterRenderer() + + input_data = {'a': 'some-data'} + + # when + actual_data = swhFilterRenderer.filter_by_fields(input_data) + + # then + self.assertEquals(actual_data, input_data) + + @patch('swh.web.ui.renderers.utils') + @patch('swh.web.ui.renderers.request') + @istest + def SWHFilterRenderer_do_filter(self, mock_request, mock_utils): + # given + mock_request.args = {'fields': 'a,c'} + mock_utils.filter_field_keys.return_value = {'a': 'some-data'} + + swhFilterRenderer = renderers.SWHFilterRenderer() + + input_data = {'a': 'some-data', + 'b': 'some-other-data'} + + # when + actual_data = swhFilterRenderer.filter_by_fields(input_data) + + # then + self.assertEquals(actual_data, {'a': 'some-data'}) + + mock_utils.filter_field_keys.assert_called_once_with(input_data, 'a,c') + + @istest + def plainRenderer(self): + # given + plainRenderer = renderers.PlainRenderer() + input_data = 'some data' + + # when + actual_data = plainRenderer.render(input_data, 'some-media-type') + + # then + self.assertEqual(actual_data, input_data) # do nothing on data + + @patch('swh.web.ui.renderers.request') + @istest + def yamlRenderer_without_filter(self, mock_request): + # given + mock_request.args = {} + yamlRenderer = renderers.YAMLRenderer() + + input_data = {'target': 'sha1-dir', + 'type': 'dir', + 'dir-id': 'dir-id-sha1-git'} + + expected_data = input_data + + # when + actual_data = yamlRenderer.render(input_data, 'application/yaml') + + # then + self.assertEqual(yaml.load(actual_data), expected_data) + + @patch('swh.web.ui.renderers.request') + @istest + def yamlRenderer(self, mock_request): + # given + mock_request.args = {'fields': 'type,target'} + yamlRenderer = renderers.YAMLRenderer() + + input_data = {'target': 'sha1-dir', + 'type': 'dir', + 'dir-id': 'dir-id-sha1-git'} + + expected_data = {'target': 'sha1-dir', 'type': 'dir'} + + # when + actual_data = yamlRenderer.render(input_data, 'application/yaml') + + # then + self.assertEqual(yaml.load(actual_data), expected_data) + + @patch('swh.web.ui.renderers.request') + @istest + def jsonRenderer_basic(self, mock_request): + # given + mock_request.args = {} + jsonRenderer = renderers.JSONPRenderer() + + input_data = {'target': 'sha1-dir', + 'type': 'dir', + 'dir-id': 'dir-id-sha1-git'} + + expected_data = input_data + + # when + actual_data = jsonRenderer.render(input_data, MediaType( + 'application/json')) + + # then + self.assertEqual(json.loads(actual_data), expected_data) + + @patch('swh.web.ui.renderers.request') + @istest + def jsonRenderer_basic_with_filter(self, mock_request): + # given + mock_request.args = {'fields': 'target'} + jsonRenderer = renderers.JSONPRenderer() + + input_data = {'target': 'sha1-dir', + 'type': 'dir', + 'dir-id': 'dir-id-sha1-git'} + + expected_data = {'target': 'sha1-dir'} + + # when + actual_data = jsonRenderer.render(input_data, MediaType( + 'application/json')) + + # then + self.assertEqual(json.loads(actual_data), expected_data) + + @patch('swh.web.ui.renderers.request') + @istest + def jsonRenderer_basic_with_filter_and_jsonp(self, mock_request): + # given + mock_request.args = {'fields': 'target', + 'callback': 'jsonpfn'} + jsonRenderer = renderers.JSONPRenderer() + + input_data = {'target': 'sha1-dir', + 'type': 'dir', + 'dir-id': 'dir-id-sha1-git'} + + # when + actual_data = jsonRenderer.render(input_data, MediaType( + 'application/json')) + + # then + self.assertEqual(actual_data, 'jsonpfn({"target": "sha1-dir"})') diff --git a/swh/web/ui/tests/test_utils.py b/swh/web/ui/tests/test_utils.py index 963ce517..49ead65c 100644 --- a/swh/web/ui/tests/test_utils.py +++ b/swh/web/ui/tests/test_utils.py @@ -1,121 +1,166 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from unittest.mock import patch from nose.tools import istest from swh.web.ui import utils class Rule(object): rule = "" endpoint = None methods = [] def __init__(self, rule, methods, endpoint): self.rule = rule self.endpoint = endpoint self.methods = methods class Map(object): _rules = [] def __init__(self, rules): self._rules = rules class UtilsTestCase(unittest.TestCase): def setUp(self): self.url_map = Map([Rule('/other/', methods=set(['GET', 'POST', 'HEAD']), endpoint='foo'), Rule('/some/old/url/', methods=set(['GET', 'POST']), endpoint='blablafn'), Rule('/other/old/url/', methods=set(['GET', 'HEAD']), endpoint='bar'), Rule('/other', methods=set([]), endpoint=None), Rule('/other2', methods=set([]), endpoint=None)]) @istest def filter_endpoints_1(self): # when actual_data = utils.filter_endpoints(self.url_map, '/some') # then self.assertEquals(actual_data, { '/some/old/url/': { 'methods': ['GET', 'POST'], 'endpoint': 'blablafn' } }) @istest def filter_endpoints_2(self): # when actual_data = utils.filter_endpoints(self.url_map, '/other', blacklist=['/other2']) # then # rules /other is skipped because its' exactly the prefix url # rules /other2 is skipped because it's blacklisted self.assertEquals(actual_data, { '/other/': { 'methods': ['GET', 'HEAD', 'POST'], 'endpoint': 'foo' }, '/other/old/url/': { 'methods': ['GET', 'HEAD'], 'endpoint': 'bar' } }) @patch('swh.web.ui.utils.flask') @istest def prepare_directory_listing(self, mock_flask): # given def mock_url_for(url_key, **kwds): if url_key == 'browse_directory': sha1_git = kwds['sha1_git'] return '/path/to/url/dir' + '/' + sha1_git else: sha1_git = kwds['q'] return '/path/to/url/file' + '/' + sha1_git mock_flask.url_for.side_effect = mock_url_for inputs = [{'type': 'dir', 'target': '123', 'name': 'some-dir-name'}, {'type': 'file', 'sha1': '654', 'name': 'some-filename'}, {'type': 'dir', 'target': '987', 'name': 'some-other-dirname'}] expected_output = [{'link': '/path/to/url/dir/123', 'name': 'some-dir-name', 'type': 'dir'}, {'link': '/path/to/url/file/654', 'name': 'some-filename', 'type': 'file'}, {'link': '/path/to/url/dir/987', 'name': 'some-other-dirname', 'type': 'dir'}] # when actual_outputs = utils.prepare_directory_listing(inputs) # then self.assertEquals(actual_outputs, expected_output) + + @istest + def filter_field_keys_dict_unknown_keys(self): + # when + actual_res = utils.filter_field_keys({'a': 1, 'c': 2, 'b': 3}, 'd') + + # then + self.assertEqual(actual_res, {}) + + @istest + def filter_field_keys_dict(self): + # when + actual_res = utils.filter_field_keys({'a': 1, 'c': 2, 'b': 3}, 'a,b') + + # then + self.assertEqual(actual_res, {'a': 1, 'b': 3}) + + @istest + def filter_field_keys_list_unknown_keys(self): + # when + actual_res = utils.filter_field_keys([{'a': 1, 'c': 2, 'b': 3}, + {'1': 1, '2': 2, 'b': 3}], 'd') + + # then + self.assertEqual(actual_res, [{}, {}]) + + @istest + def filter_field_keys_list(self): + # when + actual_res = utils.filter_field_keys([{'a': 1, 'c': 2, 'b': 3}, + {'1': 1, '2': 2, 'b': 3}], 'a,1') + + # then + self.assertEqual(actual_res, [{'a': 1}, {'1': 1}]) + + @istest + def filter_field_keys_other(self): + # given + inputSet = set([1, 2]) + + # when + actual_res = utils.filter_field_keys(inputSet, 'a,1') + + # then + self.assertEqual(actual_res, inputSet) diff --git a/swh/web/ui/utils.py b/swh/web/ui/utils.py index f6a75feb..af2ad412 100644 --- a/swh/web/ui/utils.py +++ b/swh/web/ui/utils.py @@ -1,52 +1,83 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import flask def filter_endpoints(url_map, prefix_url_rule, blacklist=[]): """Filter endpoints by prefix url rule. Args: - url_map: Url Werkzeug.Map of rules - prefix_url_rule: prefix url string - blacklist: blacklist of some url Returns: Dictionary of url_rule with values methods and endpoint. The key is the url, the associated value is a dictionary of 'methods' (possible http methods) and 'endpoint' (python function) """ out = {} for r in url_map._rules: rule = r.rule if rule == prefix_url_rule or rule in blacklist: continue if rule.startswith(prefix_url_rule): out[r.rule] = {'methods': sorted(map(str, r.methods)), 'endpoint': r.endpoint} return out def prepare_directory_listing(files): """Given a list of dictionary files, return a view ready dictionary. """ ls = [] for entry in files: new_entry = {'name': entry['name'], 'type': entry['type']} if entry['type'] == 'dir': new_entry['link'] = flask.url_for('browse_directory', sha1_git=entry['target']) else: new_entry['link'] = flask.url_for('show_content', q=entry['sha1']) ls.append(new_entry) return ls + + +def filter_field_keys(obj, field_keys): + """Given an object instance (directory or list), and a csv field keys + to filter on. + + Return the object instance with filtered keys. + + Note: Returns obj as is if it's an instance of types not in (dictionary, + list) + + Args: + - obj: one object (dictionary, list...) to filter. + - field_keys: csv or set of keys to filter the object on + + Returns: + obj filtered on field_keys + + """ + if isinstance(obj, dict): + filt_dict = {} + for key, value in obj.items(): + if key in field_keys: + filt_dict[key] = value + return filt_dict + elif isinstance(obj, list): + filt_list = [] + for e in obj: + filt_list.append(filter_field_keys(e, field_keys)) + return filt_list + return obj