diff --git a/swh/web/ui/controller.py b/swh/web/ui/controller.py index ecfe51a0..b6cf6e85 100644 --- a/swh/web/ui/controller.py +++ b/swh/web/ui/controller.py @@ -1,407 +1,191 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -from flask import render_template, request, flash, url_for +from flask import request, url_for from flask.ext.api.decorators import set_renderers -from flask.ext.api.renderers import HTMLRenderer from swh.core.hashutil import ALGORITHMS from swh.web.ui.main import app from swh.web.ui import service, renderers, utils -from swh.web.ui.exc import BadInputExc, NotFoundExc +from swh.web.ui.exc import NotFoundExc hash_filter_keys = ALGORITHMS -@app.route('/') -@set_renderers(HTMLRenderer) -def homepage(): - """Home page - - """ - flash('This Web app is still work in progress, use at your own risk', - 'warning') - # return redirect(url_for('about')) - return render_template('home.html') - - -@app.route('/about') -@set_renderers(HTMLRenderer) -def about(): - return render_template('about.html') - - -@app.route('/search') -@set_renderers(HTMLRenderer) -def search(): - """Search for hashes in swh-storage. - - """ - q = request.args.get('q', '') - env = {'q': q, 'message': ''} - - try: - if q: - r = service.lookup_hash(q) - env['message'] = 'Content with hash %s%sfound!' % ( - q, '' - if r['found'] == True else ' not ') - - except BadInputExc: - env['message'] = 'Error: invalid query string' - - return render_template('search.html', **env) - - -@app.route('/uploadnsearch', methods=['GET', 'POST']) -@set_renderers(HTMLRenderer) -def uploadnsearch(): - """Upload and search for hashes in swh-storage. - - """ - env = {'filename': None, 'message': '', 'found': None} - - if request.method == 'POST': - file = request.files['filename'] - try: - filename, sha1, found = service.upload_and_search(file) - message = 'The file %s with hash %s has%sbeen found.' % ( - filename, - sha1, - ' ' if found else ' not ') - - env.update({ - 'filename': filename, - 'sha1': sha1, - 'found': found, - 'message': message - }) - except BadInputExc: - env['message'] = 'Error: invalid query string' - - return render_template('upload_and_search.html', **env) - - -def _origin_seen(hash, data): - """Given an origin, compute a message string with the right information. - - Args: - origin: a dictionary with keys: - - origin: a dictionary with type and url keys - - occurrence: a dictionary with a validity range - - Returns: - message as a string - - """ - if data is None: - return 'Content with hash %s is unknown as of now.' % hash - - origin_type = data['origin_type'] - origin_url = data['origin_url'] - revision = data['revision'] - branch = data['branch'] - path = data['path'] - - return """The content with hash %s has been seen on origin with type '%s' -at url '%s'. The revision was identified at '%s' on branch '%s'. -The file's path referenced was '%s'.""" % (hash, - origin_type, - origin_url, - revision, - branch, - path) - - -@app.route('/browse/content/') -@set_renderers(HTMLRenderer) -def content_with_origin(q): - """Show content information. - - Args: - - q: query string of the form with - `algo_hash` in sha1, sha1_git, sha256. - - This means that several different URLs (at least one per - HASH_ALGO) will point to the same content sha: the sha with - 'hash' format - - Returns: - The content's information at for a given checksum. - - """ - env = {'q': q} - - try: - content = service.lookup_hash(q) - found = content['found'] - - if not found: - message = "Hash %s was not found." % content['algo'] - else: - origin = service.lookup_hash_origin(q) - message = _origin_seen(hash, origin) - except BadInputExc as e: # do not like it but do not duplicate code - message = e - - env['message'] = message - return render_template('content.html', **env) - - -@app.route('/browse/content//raw') -@set_renderers(HTMLRenderer) -def show_content(q): - """Given a hash and a checksum, display the content's raw data. - - Args: - q is of the form algo_hash:hash with algo_hash in - (sha1, sha1_git, sha256) - - Returns: - Information on one possible origin for such content. - - Raises: - BadInputExc in case of unknown algo_hash or bad hash - NotFoundExc if the content is not found. - - """ - env = {} - try: - content = service.lookup_content_raw(q) - if content is None: - message = 'Content with %s not found.' - - message = 'Content %s' % content['sha1'] - env['content'] = content - except BadInputExc as e: - message = e - - env['message'] = message - return render_template('display_content.html', **env) - - -def prepare_directory_listing(files): - """Given a list of dictionary files, return a view ready dictionary. - - """ - ls = [] - for entry in files: - new_entry = {} - if entry['type'] == 'dir': - new_entry['link'] = url_for('browse_directory', - sha1_git=entry['target']) - else: - new_entry['link'] = url_for('show_content', - q=entry['sha1']) - new_entry['name'] = entry['name'] - ls.append(new_entry) - - return ls - - -@app.route('/browse/directory/') -@set_renderers(HTMLRenderer) -def browse_directory(sha1_git): - """Show directory information. - - Args: - - sha1_git: the directory's sha1 git identifier. - - Returns: - The content's information at sha1_git - """ - env = {'sha1_git': sha1_git} - - try: - files = service.lookup_directory(sha1_git) - if not files: - message = "Directory %s was not found." % sha1_git - else: - message = "Listing for directory %s:" % sha1_git - env['ls'] = prepare_directory_listing(files) - - except BadInputExc as e: # do not like it but do not duplicate code - message = e - - env['message'] = message - - return render_template('directory.html', **env) - - @app.route('/api') @app.route('/api/') def api_main_points(): """List the current api endpoints starting with /api/. """ return utils.filter_endpoints(app.url_map, '/api', blacklist=['/api/']) @app.route('/api/1') @app.route('/api/1/') def api_main_points_v1(): """List the current api v1 endpoints starting with /api/. """ return utils.filter_endpoints(app.url_map, '/api/1', blacklist=['/api/1/']) @app.route('/api/1/stat/counters') def api_stats(): """Return statistics as a JSON object""" return service.stat_counters() @app.route('/api/1/search/') def api_search(q): """Return search results as a JSON object""" return {'found': service.lookup_hash(q)} def _api_lookup(criteria, lookup_fn, error_msg_if_not_found): """Factorize function regarding the api to lookup for data.""" res = lookup_fn(criteria) if not res: raise NotFoundExc(error_msg_if_not_found) return res @app.route('/api/1/origin/') def api_origin(origin_id): """Return information about origin.""" return _api_lookup( origin_id, lookup_fn=service.lookup_origin, error_msg_if_not_found='Origin with id %s not found.' % origin_id) @app.route('/api/1/person/') def api_person(person_id): """Return information about person.""" return _api_lookup( person_id, lookup_fn=service.lookup_person, error_msg_if_not_found='Person with id %s not found.' % person_id) @app.route('/api/1/release/') def api_release(sha1_git): """Return information about release with id sha1_git.""" error_msg = 'Release with sha1_git %s not found.' % sha1_git return _api_lookup( sha1_git, lookup_fn=service.lookup_release, error_msg_if_not_found=error_msg) @app.route('/api/1/revision/') def api_revision(sha1_git): """Return information about revision with id sha1_git. """ error_msg = 'Revision with sha1_git %s not found.' % sha1_git return _api_lookup( sha1_git, lookup_fn=service.lookup_revision, error_msg_if_not_found=error_msg) @app.route('/api/1/directory/') def api_directory(sha1_git): """Return information about release with id sha1_git.""" directory_entries = service.lookup_directory(sha1_git) if not directory_entries: raise NotFoundExc('Directory with sha1_git %s not found.' % sha1_git) return list(directory_entries) @app.route('/api/1/browse/') def api_content_checksum_to_origin(q): """Return content information up to one of its origin if the content is found. Args: q is of the form algo_hash:hash with algo_hash in (sha1, sha1_git, sha256) Returns: Information on one possible origin for such content. Raises: BadInputExc in case of unknown algo_hash or bad hash NotFoundExc if the content is not found. """ found = service.lookup_hash(q)['found'] if not found: raise NotFoundExc('Content with %s not found.' % q) return service.lookup_hash_origin(q) @app.route('/api/1/content//raw') @set_renderers(renderers.PlainRenderer) def api_content_raw(q): """Return content information on the content with provided hash. Args: q is of the form (algo_hash:)hash with algo_hash in (sha1, sha1_git, sha256). If no algo_hash is provided, will work with default sha1 algorithm Actual limitation: Only works with current sha1 Raises: - BadInputExc in case of unknown algo_hash or bad hash - NotFoundExc if the content is not found. """ content = service.lookup_content_raw(q) if not content: raise NotFoundExc('Content with %s not found.' % q) return content['data'] @app.route('/api/1/content/') def api_content_with_details(q): """Return content information on the content with provided hash. Args: q is of the form (algo_hash:)hash with algo_hash in (sha1, sha1_git, sha256). If no algo_hash is provided, will work with default sha1 algorithm Actual limitation: Only works with current sha1 Raises: - BadInputExc in case of unknown algo_hash or bad hash - NotFoundExc if the content is not found. """ content = service.lookup_content(q) if not content: raise NotFoundExc('Content with %s not found.' % q) content['data'] = url_for('api_content_raw', q=content['sha1']) return content @app.route('/api/1/uploadnsearch', methods=['POST']) def api_uploadnsearch(): """Upload the file's content in the post body request. Compute the hash and determine if it exists in the storage. """ file = request.files['filename'] filename, sha1, found = service.upload_and_search(file) return {'sha1': sha1, 'filename': filename, 'found': found} diff --git a/swh/web/ui/main.py b/swh/web/ui/main.py index cc9d5094..db5a49bf 100644 --- a/swh/web/ui/main.py +++ b/swh/web/ui/main.py @@ -1,128 +1,128 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import os from flask.ext.api import FlaskAPI from swh.core import config from swh.web.ui.renderers import RENDERERS DEFAULT_CONFIG = { 'storage_args': ('list[str]', ['http://localhost:5000/']), 'storage_class': ('str', 'remote_storage'), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', None), 'host': ('string', '127.0.0.1'), 'port': ('int', 6543), 'secret_key': ('string', 'development key'), 'max_upload_size': ('int', 16 * 1024 * 1024), 'upload_folder': ('string', '/tmp/swh-web-ui/uploads'), 'upload_allowed_extensions': ('list[str]', []) # means all are accepted } # api's definition app = FlaskAPI(__name__) def read_config(config_file): """Read the configuration file `config_file`, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict""" conf = config.read(config_file, DEFAULT_CONFIG) config.prepare_folders(conf, 'log_dir') if conf['storage_class'] == 'remote_storage': from swh.storage.api.client import RemoteStorage as Storage else: from swh.storage import Storage conf['storage'] = Storage(*conf['storage_args']) return conf def load_controllers(): """Load the controllers for the application""" - from swh.web.ui import controller, errorhandler # flake8: noqa + from swh.web.ui import controller, errorhandler, views # flake8: noqa def storage(): """Return the current application's storage. """ return app.config['conf']['storage'] def setup_app(app, conf): app.secret_key = conf['secret_key'] app.config['conf'] = conf app.config['MAX_CONTENT_LENGTH'] = conf['max_upload_size'] app.config['DEFAULT_RENDERERS'] = RENDERERS return app def run_from_webserver(environ, start_response): """Run the WSGI app from the webserver, loading the configuration.""" load_controllers() config_path = '/etc/softwareheritage/webapp/webapp.ini' conf = read_config(config_path) app.secret_key = conf['secret_key'] app.config['conf'] = conf app.config['MAX_CONTENT_LENGTH'] = conf['max_upload_size'] app.config['DEFAULT_RENDERERS'] = RENDERERS logging.basicConfig(filename=os.path.join(conf['log_dir'], 'web-ui.log'), level=logging.INFO) return app(environ, start_response) def run_debug_from(config_path, verbose=False): """Run the api's server in dev mode. Args: conf is a dictionary of keywords: - 'db_url' the db url's access (through psycopg2 format) - 'content_storage_dir' revisions/directories/contents storage on disk - 'host' to override the default 127.0.0.1 to open or not the server to the world - 'port' to override the default of 5000 (from the underlying layer: flask) - 'debug' activate the verbose logs - 'secret_key' the flask secret key Returns: Never """ load_controllers() conf = read_config(config_path) app.secret_key = conf['secret_key'] app.config['conf'] = conf app.config['MAX_CONTENT_LENGTH'] = conf['max_upload_size'] app.config['DEFAULT_RENDERERS'] = RENDERERS host = conf.get('host', '127.0.0.1') port = conf.get('port') debug = conf.get('debug') log_file = os.path.join(conf['log_dir'], 'web-ui.log') logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO, handlers=[logging.FileHandler(log_file), logging.StreamHandler()]) app.run(host=host, port=port, debug=debug) diff --git a/swh/web/ui/controller.py b/swh/web/ui/views.py similarity index 55% copy from swh/web/ui/controller.py copy to swh/web/ui/views.py index ecfe51a0..e6455268 100644 --- a/swh/web/ui/controller.py +++ b/swh/web/ui/views.py @@ -1,407 +1,233 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -from flask import render_template, request, flash, url_for +from flask import render_template, flash, request, url_for from flask.ext.api.decorators import set_renderers from flask.ext.api.renderers import HTMLRenderer from swh.core.hashutil import ALGORITHMS +from swh.web.ui import service +from swh.web.ui.exc import BadInputExc from swh.web.ui.main import app -from swh.web.ui import service, renderers, utils -from swh.web.ui.exc import BadInputExc, NotFoundExc hash_filter_keys = ALGORITHMS @app.route('/') @set_renderers(HTMLRenderer) def homepage(): """Home page """ flash('This Web app is still work in progress, use at your own risk', 'warning') # return redirect(url_for('about')) return render_template('home.html') @app.route('/about') @set_renderers(HTMLRenderer) def about(): return render_template('about.html') @app.route('/search') @set_renderers(HTMLRenderer) def search(): """Search for hashes in swh-storage. """ q = request.args.get('q', '') env = {'q': q, 'message': ''} try: if q: r = service.lookup_hash(q) env['message'] = 'Content with hash %s%sfound!' % ( q, '' if r['found'] == True else ' not ') except BadInputExc: env['message'] = 'Error: invalid query string' return render_template('search.html', **env) @app.route('/uploadnsearch', methods=['GET', 'POST']) @set_renderers(HTMLRenderer) def uploadnsearch(): """Upload and search for hashes in swh-storage. """ env = {'filename': None, 'message': '', 'found': None} if request.method == 'POST': file = request.files['filename'] try: filename, sha1, found = service.upload_and_search(file) message = 'The file %s with hash %s has%sbeen found.' % ( filename, sha1, ' ' if found else ' not ') env.update({ 'filename': filename, 'sha1': sha1, 'found': found, 'message': message }) except BadInputExc: env['message'] = 'Error: invalid query string' return render_template('upload_and_search.html', **env) def _origin_seen(hash, data): """Given an origin, compute a message string with the right information. Args: origin: a dictionary with keys: - origin: a dictionary with type and url keys - occurrence: a dictionary with a validity range Returns: message as a string """ if data is None: return 'Content with hash %s is unknown as of now.' % hash origin_type = data['origin_type'] origin_url = data['origin_url'] revision = data['revision'] branch = data['branch'] path = data['path'] return """The content with hash %s has been seen on origin with type '%s' at url '%s'. The revision was identified at '%s' on branch '%s'. The file's path referenced was '%s'.""" % (hash, origin_type, origin_url, revision, branch, path) @app.route('/browse/content/') @set_renderers(HTMLRenderer) def content_with_origin(q): """Show content information. Args: - q: query string of the form with `algo_hash` in sha1, sha1_git, sha256. This means that several different URLs (at least one per HASH_ALGO) will point to the same content sha: the sha with 'hash' format Returns: The content's information at for a given checksum. """ env = {'q': q} try: content = service.lookup_hash(q) found = content['found'] if not found: message = "Hash %s was not found." % content['algo'] else: origin = service.lookup_hash_origin(q) message = _origin_seen(hash, origin) except BadInputExc as e: # do not like it but do not duplicate code message = e env['message'] = message return render_template('content.html', **env) @app.route('/browse/content//raw') @set_renderers(HTMLRenderer) def show_content(q): """Given a hash and a checksum, display the content's raw data. Args: q is of the form algo_hash:hash with algo_hash in (sha1, sha1_git, sha256) Returns: Information on one possible origin for such content. Raises: BadInputExc in case of unknown algo_hash or bad hash NotFoundExc if the content is not found. """ env = {} try: content = service.lookup_content_raw(q) if content is None: message = 'Content with %s not found.' message = 'Content %s' % content['sha1'] env['content'] = content except BadInputExc as e: message = e env['message'] = message return render_template('display_content.html', **env) def prepare_directory_listing(files): """Given a list of dictionary files, return a view ready dictionary. """ ls = [] for entry in files: new_entry = {} if entry['type'] == 'dir': new_entry['link'] = url_for('browse_directory', sha1_git=entry['target']) else: new_entry['link'] = url_for('show_content', q=entry['sha1']) new_entry['name'] = entry['name'] ls.append(new_entry) return ls @app.route('/browse/directory/') @set_renderers(HTMLRenderer) def browse_directory(sha1_git): """Show directory information. Args: - sha1_git: the directory's sha1 git identifier. Returns: The content's information at sha1_git """ env = {'sha1_git': sha1_git} try: files = service.lookup_directory(sha1_git) if not files: message = "Directory %s was not found." % sha1_git else: message = "Listing for directory %s:" % sha1_git env['ls'] = prepare_directory_listing(files) except BadInputExc as e: # do not like it but do not duplicate code message = e env['message'] = message return render_template('directory.html', **env) - - -@app.route('/api') -@app.route('/api/') -def api_main_points(): - """List the current api endpoints starting with /api/. - - """ - return utils.filter_endpoints(app.url_map, '/api', blacklist=['/api/']) - - -@app.route('/api/1') -@app.route('/api/1/') -def api_main_points_v1(): - """List the current api v1 endpoints starting with /api/. - - """ - return utils.filter_endpoints(app.url_map, '/api/1', - blacklist=['/api/1/']) - - -@app.route('/api/1/stat/counters') -def api_stats(): - """Return statistics as a JSON object""" - return service.stat_counters() - - -@app.route('/api/1/search/') -def api_search(q): - """Return search results as a JSON object""" - return {'found': service.lookup_hash(q)} - - -def _api_lookup(criteria, lookup_fn, error_msg_if_not_found): - """Factorize function regarding the api to lookup for data.""" - res = lookup_fn(criteria) - if not res: - raise NotFoundExc(error_msg_if_not_found) - return res - - -@app.route('/api/1/origin/') -def api_origin(origin_id): - """Return information about origin.""" - return _api_lookup( - origin_id, lookup_fn=service.lookup_origin, - error_msg_if_not_found='Origin with id %s not found.' % origin_id) - - -@app.route('/api/1/person/') -def api_person(person_id): - """Return information about person.""" - return _api_lookup( - person_id, lookup_fn=service.lookup_person, - error_msg_if_not_found='Person with id %s not found.' % person_id) - - -@app.route('/api/1/release/') -def api_release(sha1_git): - """Return information about release with id sha1_git.""" - error_msg = 'Release with sha1_git %s not found.' % sha1_git - return _api_lookup( - sha1_git, - lookup_fn=service.lookup_release, - error_msg_if_not_found=error_msg) - - -@app.route('/api/1/revision/') -def api_revision(sha1_git): - """Return information about revision with id sha1_git. - - """ - error_msg = 'Revision with sha1_git %s not found.' % sha1_git - return _api_lookup( - sha1_git, - lookup_fn=service.lookup_revision, - error_msg_if_not_found=error_msg) - - -@app.route('/api/1/directory/') -def api_directory(sha1_git): - """Return information about release with id sha1_git.""" - directory_entries = service.lookup_directory(sha1_git) - if not directory_entries: - raise NotFoundExc('Directory with sha1_git %s not found.' % sha1_git) - return list(directory_entries) - - -@app.route('/api/1/browse/') -def api_content_checksum_to_origin(q): - """Return content information up to one of its origin if the content - is found. - - Args: - q is of the form algo_hash:hash with algo_hash in - (sha1, sha1_git, sha256) - - Returns: - Information on one possible origin for such content. - - Raises: - BadInputExc in case of unknown algo_hash or bad hash - NotFoundExc if the content is not found. - - """ - found = service.lookup_hash(q)['found'] - if not found: - raise NotFoundExc('Content with %s not found.' % q) - - return service.lookup_hash_origin(q) - - -@app.route('/api/1/content//raw') -@set_renderers(renderers.PlainRenderer) -def api_content_raw(q): - """Return content information on the content with provided hash. - - Args: - q is of the form (algo_hash:)hash with algo_hash in - (sha1, sha1_git, sha256). - If no algo_hash is provided, will work with default sha1 - algorithm - - Actual limitation: - Only works with current sha1 - - Raises: - - BadInputExc in case of unknown algo_hash or bad hash - - NotFoundExc if the content is not found. - - """ - content = service.lookup_content_raw(q) - if not content: - raise NotFoundExc('Content with %s not found.' % q) - - return content['data'] - - -@app.route('/api/1/content/') -def api_content_with_details(q): - """Return content information on the content with provided hash. - - Args: - q is of the form (algo_hash:)hash with algo_hash in - (sha1, sha1_git, sha256). - If no algo_hash is provided, will work with default sha1 - algorithm - - Actual limitation: - Only works with current sha1 - - Raises: - - BadInputExc in case of unknown algo_hash or bad hash - - NotFoundExc if the content is not found. - - """ - content = service.lookup_content(q) - if not content: - raise NotFoundExc('Content with %s not found.' % q) - - content['data'] = url_for('api_content_raw', q=content['sha1']) - return content - - -@app.route('/api/1/uploadnsearch', methods=['POST']) -def api_uploadnsearch(): - """Upload the file's content in the post body request. - Compute the hash and determine if it exists in the storage. - """ - file = request.files['filename'] - filename, sha1, found = service.upload_and_search(file) - return {'sha1': sha1, - 'filename': filename, - 'found': found}