diff --git a/swh/web/ui/api.py b/swh/web/ui/api.py index 7dc96fad..d538475f 100644 --- a/swh/web/ui/api.py +++ b/swh/web/ui/api.py @@ -1,320 +1,310 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from flask import request, url_for, Response from swh.web.ui import service from swh.web.ui.exc import BadInputExc, NotFoundExc from swh.web.ui.main import app -def install_browsable_api_endpoints(): - for doc_url_rule in ['/browse/', '/api/', '/api/1/']: - endpoint_name = doc_url_rule.strip('/').replace('/', '_') - app.add_url_rule(doc_url_rule, - endpoint=endpoint_name, - view_func=lambda: utils.filter_endpoints( - main.rules(), - doc_url_rule)) - - @app.route('/api/1/stat/counters/') def api_stats(): """Return statistics on SWH storage. Returns: SWH storage's statistics """ return service.stat_counters() @app.route('/api/1/search/') @app.route('/api/1/search//') def api_search(q='sha1:bd819b5b28fcde3bf114d16a44ac46250da94ee5'): """Search a content per hash. Args: q is of the form algo_hash:hash with algo_hash in (sha1, sha1_git, sha256) Returns: Dictionary with 'found' key and the associated result. Raises: BadInputExc in case of unknown algo_hash or bad hash Example: GET /api/1/search/sha1:bd819b5b28fcde3bf114d16a44ac46250da94ee5/ """ r = service.lookup_hash(q).get('found') return {'found': True if r else False} def _api_lookup(criteria, lookup_fn, error_msg_if_not_found): """Factorize function regarding the api to lookup for data.""" res = lookup_fn(criteria) if not res: raise NotFoundExc(error_msg_if_not_found) if isinstance(res, map): return list(res) return res @app.route('/api/1/origin/') @app.route('/api/1/origin//') def api_origin(origin_id=1): """Return information about origin with id origin_id. Args: origin_id: the origin's identifier Returns: Information on the origin if found. Raises: NotFoundExc if the origin is not found. Example: GET /api/1/origin/1/ """ return _api_lookup( origin_id, lookup_fn=service.lookup_origin, error_msg_if_not_found='Origin with id %s not found.' % origin_id) @app.route('/api/1/person/') @app.route('/api/1/person//') def api_person(person_id=1): """Return information about person with identifier person_id. Args: person_id: the person's identifier Returns: Information on the person if found. Raises: NotFoundExc if the person is not found. Example: GET /api/1/person/1/ """ return _api_lookup( person_id, lookup_fn=service.lookup_person, error_msg_if_not_found='Person with id %s not found.' % person_id) @app.route('/api/1/release/') @app.route('/api/1/release//') def api_release(sha1_git='b307094f00c3641b0c9da808d894f3a325371414'): """Return information about release with id sha1_git. Args: sha1_git: the release's hash Returns: Information on the release if found. Raises: BadInputExc in case of unknown algo_hash or bad hash NotFoundExc if the release is not found. Example: GET /api/1/release/b307094f00c3641b0c9da808d894f3a325371414 """ error_msg = 'Release with sha1_git %s not found.' % sha1_git return _api_lookup( sha1_git, lookup_fn=service.lookup_release, error_msg_if_not_found=error_msg) @app.route('/api/1/revision/') @app.route('/api/1/revision//') def api_revision(sha1_git='baf18f9fc50a0b6fef50460a76c33b2ddc57486e'): """Return information about revision with id sha1_git. Args: sha1_git: the revision's hash Returns: Information on the revision if found. Raises: BadInputExc in case of unknown algo_hash or bad hash NotFoundExc if the revision is not found. Example: GET /api/1/revision/baf18f9fc50a0b6fef50460a76c33b2ddc57486e """ error_msg = 'Revision with sha1_git %s not found.' % sha1_git return _api_lookup( sha1_git, lookup_fn=service.lookup_revision, error_msg_if_not_found=error_msg) @app.route('/api/1/revision//log/') def api_revision_log(sha1_git): """Show all revisions (~git log) starting from sha1_git. The first element returned is the given sha1_git. Args: sha1_git: the revision's hash Returns: Information on the revision if found. Raises: BadInputExc in case of unknown algo_hash or bad hash NotFoundExc if the revision is not found. """ error_msg = 'Revision with sha1_git %s not found.' % sha1_git return _api_lookup( sha1_git, lookup_fn=service.lookup_revision_log, error_msg_if_not_found=error_msg) @app.route('/api/1/directory/') @app.route('/api/1/directory//') def api_directory(sha1_git='8d7dc91d18546a91564606c3e3695a5ab568d179'): """Return information about release with id sha1_git. Args: Directory's sha1_git Raises: BadInputExc in case of unknown algo_hash or bad hash NotFoundExc if the content is not found. Example: GET /api/1/directory/8d7dc91d18546a91564606c3e3695a5ab568d179 """ error_msg = 'Directory with sha1_git %s not found.' % sha1_git return _api_lookup( sha1_git, lookup_fn=service.lookup_directory, error_msg_if_not_found=error_msg) @app.route('/api/1/browse/') @app.route('/api/1/browse//') def api_content_checksum_to_origin(q='sha1_git:88b9b366facda0b5ff8d8640ee9279b' 'ed346f242'): """Return content information up to one of its origin if the content is found. Args: q is of the form algo_hash:hash with algo_hash in (sha1, sha1_git, sha256) Returns: Information on one possible origin for such content. Raises: BadInputExc in case of unknown algo_hash or bad hash NotFoundExc if the content is not found. Example: GET /api/1/browse/sha1_git:88b9b366facda0b5ff8d8640ee9279bed346f242 """ found = service.lookup_hash(q)['found'] if not found: raise NotFoundExc('Content with %s not found.' % q) return service.lookup_hash_origin(q) @app.route('/api/1/content//raw/') def api_content_raw(q): """Return content's raw data if content is found. Args: q is of the form (algo_hash:)hash with algo_hash in (sha1, sha1_git, sha256). When algo_hash is not provided, 'hash' is considered sha1. Returns: Content's raw data in application/octet-stream Raises: - BadInputExc in case of unknown algo_hash or bad hash - NotFoundExc if the content is not found. """ def generate(content): yield content['data'] content = service.lookup_content_raw(q) if not content: raise NotFoundExc('Content with %s not found.' % q) return Response(generate(content), mimetype='application/octet-stream') @app.route('/api/1/content/') @app.route('/api/1/content//') def api_content_with_details(q='sha256:bc872dd166e4ef82b4b3dbf65f8a974c21e6aa' '16c99b2c725594a470eeed37c7'): """Return content information if content is found. Args: q is of the form (algo_hash:)hash with algo_hash in (sha1, sha1_git, sha256). When algo_hash is not provided, 'hash' is considered sha1. Returns: Content's information. Raises: - BadInputExc in case of unknown algo_hash or bad hash - NotFoundExc if the content is not found. Example: GET /api/1/content/sha256:bc872dd166e4ef82b4b3dbf65f8a974c21e6aa16c99 b2c725594a470eeed37c7 """ content = service.lookup_content(q) if not content: raise NotFoundExc('Content with %s not found.' % q) content['data'] = url_for('api_content_raw', q=content['sha1']) return content @app.route('/api/1/uploadnsearch/', methods=['POST']) def api_uploadnsearch(): """Upload the file's content in the post body request. Compute its hash and determine if it exists in the storage. Args: request.files filled with the filename's data to upload. Returns: Dictionary with 'sha1', 'filename' and 'found' predicate depending on whether we find it or not. Raises: BadInputExc in case of the form submitted is incorrect. """ file = request.files.get('filename') if not file: raise BadInputExc('Bad request, missing \'filename\' entry in form.') return service.upload_and_search(file) diff --git a/swh/web/ui/main.py b/swh/web/ui/main.py index 073f60d7..7893a1ab 100644 --- a/swh/web/ui/main.py +++ b/swh/web/ui/main.py @@ -1,142 +1,184 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import os from flask.ext.api import FlaskAPI from swh.core import config +from swh.web.ui import utils from swh.web.ui.renderers import RENDERERS DEFAULT_CONFIG = { 'storage_args': ('list[str]', ['http://localhost:5000/']), 'storage_class': ('str', 'remote_storage'), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', None), 'host': ('string', '127.0.0.1'), 'port': ('int', 6543), 'secret_key': ('string', 'development key'), 'max_upload_size': ('int', 16 * 1024 * 1024), 'upload_folder': ('string', '/tmp/swh-web-ui/uploads'), 'upload_allowed_extensions': ('list[str]', []) # means all are accepted } # api's definition app = FlaskAPI(__name__) def read_config(config_file): """Read the configuration file `config_file`, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict""" conf = config.read(config_file, DEFAULT_CONFIG) config.prepare_folders(conf, 'log_dir', 'upload_folder') if conf['storage_class'] == 'remote_storage': from swh.storage.api.client import RemoteStorage as Storage else: from swh.storage import Storage conf['storage'] = Storage(*conf['storage_args']) return conf def load_controllers(): """Load the controllers for the application""" from swh.web.ui import api, errorhandler, views, apidoc # flake8: noqa - api.install_browsable_api_endpoints() + install_browsable_api_endpoints() def rules(): """Returns rules from the application in dictionary form. Beware, must be called after swh.web.ui.main.load_controllers funcall. Returns: Generator of application's rules. """ for rule in app.url_map._rules: yield {'rule': rule.rule, 'methods': rule.methods, 'endpoint': rule.endpoint} +def _create_url_doc_endpoints(rules): + def split_path(path, acc): + rpath = os.path.dirname(path) + if rpath == '/': + yield from acc + else: + acc.append(rpath+'/') + yield from split_path(rpath, acc) + + url_doc_endpoints = set() + for rule in rules: + url_rule = rule['rule'] + url_doc_endpoints.add(url_rule) + if '<' in rule or '>' in url_rule: + continue + acc = [] + for rpath in split_path(url_rule, acc): + if rpath in url_doc_endpoints: + continue + yield rpath + url_doc_endpoints.add(rpath) + + +def install_browsable_api_endpoints(): + """Install browsable endpoints. + + """ + url_doc_endpoints = _create_url_doc_endpoints(rules()) + # url_doc_endpoints = ['/api/1/stat/', '/api/', '/api/1/'] + for url_doc in url_doc_endpoints: + endpoint_name = 'doc_api_' + url_doc.strip('/').replace('/', '_') + view_func = lambda : utils.filter_endpoints(rules(), url_doc) +# print(url_doc, endpoint_name, view_func) + app.add_url_rule(rule=url_doc, + endpoint=endpoint_name, + view_func=view_func, + methods=['GET']) + +# print(app.url_map._rules) + + def storage(): """Return the current application's storage. """ return app.config['conf']['storage'] def setup_app(app, conf): app.secret_key = conf['secret_key'] app.config['conf'] = conf app.config['MAX_CONTENT_LENGTH'] = conf['max_upload_size'] app.config['DEFAULT_RENDERERS'] = RENDERERS return app def run_from_webserver(environ, start_response): """Run the WSGI app from the webserver, loading the configuration.""" load_controllers() config_path = '/etc/softwareheritage/webapp/webapp.ini' conf = read_config(config_path) app.secret_key = conf['secret_key'] app.config['conf'] = conf app.config['MAX_CONTENT_LENGTH'] = conf['max_upload_size'] app.config['DEFAULT_RENDERERS'] = RENDERERS logging.basicConfig(filename=os.path.join(conf['log_dir'], 'web-ui.log'), level=logging.INFO) return app(environ, start_response) def run_debug_from(config_path, verbose=False): """Run the api's server in dev mode. Args: conf is a dictionary of keywords: - 'db_url' the db url's access (through psycopg2 format) - 'content_storage_dir' revisions/directories/contents storage on disk - 'host' to override the default 127.0.0.1 to open or not the server to the world - 'port' to override the default of 5000 (from the underlying layer: flask) - 'debug' activate the verbose logs - 'secret_key' the flask secret key Returns: Never """ load_controllers() conf = read_config(config_path) app.secret_key = conf['secret_key'] app.config['conf'] = conf app.config['MAX_CONTENT_LENGTH'] = conf['max_upload_size'] app.config['DEFAULT_RENDERERS'] = RENDERERS host = conf.get('host', '127.0.0.1') port = conf.get('port') debug = conf.get('debug') log_file = os.path.join(conf['log_dir'], 'web-ui.log') logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO, handlers=[logging.FileHandler(log_file), logging.StreamHandler()]) app.run(host=host, port=port, debug=debug)