diff --git a/swh/web/ui/controller.py b/swh/web/ui/controller.py index 10a3b37c..b7cb3225 100644 --- a/swh/web/ui/controller.py +++ b/swh/web/ui/controller.py @@ -1,377 +1,384 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json from flask import render_template, jsonify, request, flash from flask import make_response from swh.core.hashutil import ALGORITHMS from swh.web.ui.main import app from swh.web.ui import service from swh.web.ui.decorators import jsonp hash_filter_keys = ALGORITHMS @app.route('/') def main(): """Home page """ flash('This Web app is still work in progress, use at your own risk', 'warning') # return redirect(url_for('about')) return render_template('home.html') @app.route('/about') def about(): return render_template('about.html') @app.route('/search') def search(): """Search for hashes in swh-storage. """ q = request.args.get('q', '') env = {'q': q, 'message': '', 'found': None} try: if q: env['found'] = service.lookup_hash(q) except ValueError: env['message'] = 'Error: invalid query string' return render_template('search.html', **env) @app.route('/uploadnsearch', methods=['GET', 'POST']) def uploadnsearch(): """Upload and search for hashes in swh-storage. """ env = {'filename': None, 'message': '', 'found': None} if request.method == 'POST': file = request.files['filename'] try: filename, sha1, found = service.upload_and_search(file) message = 'The file %s with hash %s has%sbeen found.' % ( filename, sha1, ' ' if found else ' not ') env.update({ 'filename': filename, 'sha1': sha1, 'found': found, 'message': message }) except ValueError: env['message'] = 'Error: invalid query string' return render_template('upload_and_search.html', **env) @app.route('/browse/revision/') def revision(sha1_git): """Show commit information. Args: sha1_git: the revision's sha1 Returns: Revision information """ return render_template('revision.html', sha1_git=sha1_git) @app.route('/browse/directory/') def directory(sha1_git): """Show directory information. Args: sha1_git: the directory's sha1 Returns: Directory information """ return render_template('directory.html', sha1_git=sha1_git) @app.route('/browse/directory//') def directory_at_path(sha1_git, p): """Show directory information for the sha1_git at path. Args: sha1_git: the directory's sha1 path: file or directory pointed to Returns: Directory information at sha1_git + path """ return render_template('directory.html', sha1_git=sha1_git, path=p) def _origin_seen(hash, data): """Given an origin, compute a message string with the right information. Args: origin: a dictionary with keys: - origin: a dictionary with type and url keys - occurrence: a dictionary with a validity range Returns: message as a string """ if data is None: return 'Content with hash %s is unknown as of now.' % hash origin_type = data['origin_type'] origin_url = data['origin_url'] revision = data['revision'] branch = data['branch'] path = data['path'] return """The content with hash %s has been seen on origin with type '%s' at url '%s'. The revision was identified at '%s' on branch '%s'. The file's path referenced was '%s'.""" % (hash, origin_type, origin_url, revision, branch, path) @app.route('/browse/content/:') def content(hash, sha): """Show content information. Args: hash: hash according to HASH_ALGO, where HASH_ALGO is one of: sha1, sha1_git, sha256. This means that several different URLs (at least one per HASH_ALGO) will point to the same content sha: the sha with 'hash' format Returns: The content's information at sha1_git """ if hash not in hash_filter_keys: message = 'The checksum must be one of sha1, sha1_git, sha256' else: q = "%s:%s" % (hash, sha) found = service.lookup_hash(q) if not found: message = "Hash %s was not found." % hash else: origin = service.lookup_hash_origin(q) message = _origin_seen(hash, origin) return render_template('content.html', hash=hash, sha=sha, message=message) @app.route('/browse/release/') def release(sha1_git): """Show release's information. Args: sha1_git: sha1_git for this particular release Returns: Release's information """ return 'Release information at %s' % sha1_git @app.route('/browse/person/') def person(id): """Show Person's information at id. Args: id: person's unique identifier Returns: Person's information """ return 'Person information at %s' % id @app.route('/browse/origin/') def origin(id): """Show origin's information at id. Args: id: origin's unique identifier Returns: Origin's information """ return 'Origin information at %s' % id @app.route('/browse/project/') def project(id): """Show project's information at id. Args: id: project's unique identifier Returns: Project's information """ return 'Project information at %s' % id @app.route('/browse/organization/') def organization(id): """Show organization's information at id. Args: id: organization's unique identifier Returns: Organization's information """ return 'Organization information at %s' % id @app.route('/browse/directory//' '+|/' '|/') def directory_at_origin(timestamp, origin_type, origin_url, branch, path): """Show directory information at timestamp, origin-type, origin-url, branch and path. Those parameters are separated by the `|` terminator. Args: timestamp: the timestamp to look for. can be latest or some iso8601 date format. (TODO: decide the time matching policy.) origin_type: origin's type origin_url: origin's url (can contain `/`) branch: branch name which can contain `/` path: path to directory or file Returns: Directory information at the given parameters. """ return 'Directory at (%s, %s, %s, %s, %s)' % (timestamp, origin_type, origin_url, branch, path) @app.route('/browse/revision//' '+|/') def revision_at_origin_and_branch(timestamp, origin_type, origin_url, branch): """Show revision information at timestamp, origin, and branch. Those parameters are separated by the `|` terminator. Args: timestamp: the timestamp to look for. can be latest or some iso8601 date format. (TODO: decide the time matching policy.) origin_type: origin's type origin_url: origin's url (can contain `/`) branch: branch name which can contain / Returns: Revision information at the given parameters. """ return 'Revision at (ts=%s, type=%s, url=%s, branch=%s)' % (timestamp, origin_type, origin_url, branch) @app.route('/browse/revision//' '+|') def revision_at_origin(timestamp, origin_type, origin_url): """Show revision information at timestamp, origin, and branch. Those parameters are separated by the `|` terminator. Args: timestamp: the timestamp to look for. can be latest or iso8601 date format. (TODO: decide the time matching policy.) origin_type: origin's type origin_url: origin's url (can contain `/`) Returns: Revision information at the given parameters. """ return 'Revision at (timestamp=%s, type=%s, url=%s)' % (timestamp, origin_type, origin_url) @app.route('/api/1/stat/counters') @jsonp def api_stats(): """Return statistics as a JSON object""" return jsonify(service.stat_counters()) @app.errorhandler(ValueError) def value_error_as_bad_request(error): """Compute a bad request and add body as payload. """ response = make_response( 'Bad request', 400) response.headers['Content-Type'] = 'application/json' response.data = json.dumps({"error": str(error)}) return response @app.route('/api/1/search//') @jsonp def api_search(q): """Return search results as a JSON object""" return jsonify({'found': service.lookup_hash(q)}) +@app.route('/api/1/origin/') +@jsonp +def api_origin(origin_id): + """Return information about origin""" + return jsonify({'origin': service.lookup_origin(origin_id)}) + + @app.route('/api/1/browse//') @jsonp def api_browse(q): """Return search results as a JSON object""" return jsonify({'origin': service.lookup_hash_origin(q)}) @app.route('/api/1/uploadnsearch/', methods=['POST']) @jsonp def api_uploadnsearch(): """Upload the file's content in the post body request. Compute the hash and determine if it exists in the storage. """ file = request.files['filename'] filename, sha1, found = service.upload_and_search(file) return jsonify({'sha1': sha1, 'filename': filename, 'found': found}) diff --git a/swh/web/ui/service.py b/swh/web/ui/service.py index 718fda0e..de134ad2 100644 --- a/swh/web/ui/service.py +++ b/swh/web/ui/service.py @@ -1,78 +1,91 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.core import hashutil from swh.web.ui import converters, main, query, upload def hash_and_search(filepath): """Hash the filepath's content as sha1, then search in storage if it exists. Args: Filepath of the file to hash and search. Returns: Tuple (hex sha1, found as True or false). The found boolean, according to whether the sha1 of the file is present or not. """ hash = hashutil.hashfile(filepath) return (hashutil.hash_to_hex(hash['sha1']), main.storage().content_exist(hash)) def upload_and_search(file): """Upload a file and compute its hash. """ tmpdir, filename, filepath = upload.save_in_upload_folder(file) try: sha1, found = None, None if filepath: sha1, found = hash_and_search(filepath) return filename, sha1, found finally: # clean up if tmpdir: upload.cleanup(tmpdir) def lookup_hash(q): """Checks if the storage contains a given content checksum Args: query string Returns: True or False, according to whether the checksum is present or not """ (algo, hash) = query.parse_hash(q) return main.storage().content_exist({algo: hash}) def lookup_hash_origin(q): """Return information about the checksum contained in the query q. Args: query string Returns: True or False, according to whether the checksum is present or not """ algo, h = query.parse_hash(q) origin = main.storage().content_find_occurrence({algo: h}) return converters.from_origin(origin) +def lookup_origin(origin_id): + """Return information about the origin with id origin_id. + + Args: + origin_id as string + + Returns: + origin information as dict. + + """ + return main.storage().origin_get({'id': origin_id}) + + def stat_counters(): """Return the stat counters for Software Heritage Returns: A dict mapping textual labels to integer values. """ return main.storage().stat_counters() diff --git a/swh/web/ui/tests/test_controller.py b/swh/web/ui/tests/test_controller.py index dc9d9088..c50c1df3 100644 --- a/swh/web/ui/tests/test_controller.py +++ b/swh/web/ui/tests/test_controller.py @@ -1,165 +1,194 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest import json from nose.tools import istest from unittest.mock import patch from swh.web.ui.tests import test_app class ApiTestCase(unittest.TestCase): @classmethod def setUpClass(cls): cls.app, _, _ = test_app.init_app() @istest def info(self): # when rv = self.app.get('/about') self.assertEquals(rv.status_code, 200) self.assertIn(b'About', rv.data) # @istest def search_1(self): # when rv = self.app.get('/search') self.assertEquals(rv.status_code, 200) # check this api self.assertRegexpMatches(rv.data, b'name=q value=>') # @istest def search_2(self): # when rv = self.app.get('/search?q=one-hash-to-look-for:another-one') self.assertEquals(rv.status_code, 200) # check this api self.assertRegexpMatches( rv.data, b'name=q value=one-hash-to-look-for:another-one') @patch('swh.web.ui.controller.service') @istest def api_browse(self, mock_service): # given mock_service.lookup_hash_origin.return_value = { 'origin': 'some-origin' } # when rv = self.app.get('/api/1/browse/sha1:foo/') self.assertEquals(rv.status_code, 200) self.assertEquals(rv.mimetype, 'application/json') response_data = json.loads(rv.data.decode('utf-8')) self.assertEquals(response_data, {'origin': {'origin': 'some-origin'}}) mock_service.lookup_hash_origin.assert_called_once_with('sha1:foo') @patch('swh.web.ui.controller.service') @istest def api_search(self, mock_service): # given mock_service.lookup_hash.return_value = False # when rv = self.app.get('/api/1/search/sha1:blah/') self.assertEquals(rv.status_code, 200) self.assertEquals(rv.mimetype, 'application/json') response_data = json.loads(rv.data.decode('utf-8')) self.assertEquals(response_data, {'found': False}) mock_service.lookup_hash.assert_called_once_with('sha1:blah') @patch('swh.web.ui.controller.service') @istest def api_1_stat_counters_raise_error(self, mock_service): # given mock_service.stat_counters.side_effect = ValueError( 'voluntary error to check the bad request middleware.') # when rv = self.app.get('/api/1/stat/counters') # then self.assertEquals(rv.status_code, 400) self.assertEquals(rv.mimetype, 'application/json') response_data = json.loads(rv.data.decode('utf-8')) self.assertEquals(response_data, { 'error': 'voluntary error to check the bad request middleware.'}) @patch('swh.web.ui.controller.service') @istest def api_1_stat_counters(self, mock_service): # given mock_service.stat_counters.return_value = { "content": 1770830, "directory": 211683, "directory_entry_dir": 209167, "directory_entry_file": 1807094, "directory_entry_rev": 0, "entity": 0, "entity_history": 0, "occurrence": 0, "occurrence_history": 19600, "origin": 1096, "person": 0, "release": 8584, "revision": 7792, "revision_history": 0, "skipped_content": 0 } # when rv = self.app.get('/api/1/stat/counters') response_data = json.loads(rv.data.decode('utf-8')) self.assertEquals(rv.status_code, 200) self.assertEquals(rv.mimetype, 'application/json') self.assertEquals(response_data, { "content": 1770830, "directory": 211683, "directory_entry_dir": 209167, "directory_entry_file": 1807094, "directory_entry_rev": 0, "entity": 0, "entity_history": 0, "occurrence": 0, "occurrence_history": 19600, "origin": 1096, "person": 0, "release": 8584, "revision": 7792, "revision_history": 0, "skipped_content": 0 }) mock_service.stat_counters.assert_called_once_with() @patch('swh.web.ui.controller.service') @patch('swh.web.ui.controller.request') @istest def api_uploadnsearch(self, mock_request, mock_service): # given mock_request.files = {'filename': 'simple-filename'} mock_service.upload_and_search.return_value = ( 'simple-filename', 'some-hex-sha1', False) # when rv = self.app.post('/api/1/uploadnsearch/') self.assertEquals(rv.status_code, 200) self.assertEquals(rv.mimetype, 'application/json') response_data = json.loads(rv.data.decode('utf-8')) self.assertEquals(response_data, {'filename': 'simple-filename', 'sha1': 'some-hex-sha1', 'found': False}) mock_service.upload_and_search.assert_called_once_with( 'simple-filename') + + @patch('swh.web.ui.controller.service') + @istest + def api_origin(self, mock_service): + # given + mock_service.lookup_origin.return_value = { + 'id': 'origin-0', + 'lister': 'uuid-lister-0', + 'project': 'uuid-project-0', + 'url': 'ftp://some/url/to/origin/0', + 'type': 'ftp'} + + # when + rv = self.app.get('/api/1/origin/origin-0') + + # then + self.assertEquals(rv.status_code, 200) + self.assertEquals(rv.mimetype, 'application/json') + + response_data = json.loads(rv.data.decode('utf-8')) + self.assertEquals(response_data, { + 'origin': { + 'id': 'origin-0', + 'lister': 'uuid-lister-0', + 'project': 'uuid-project-0', + 'url': 'ftp://some/url/to/origin/0', + 'type': 'ftp' + } + }) diff --git a/swh/web/ui/tests/test_service.py b/swh/web/ui/tests/test_service.py index 5a355a57..b6a9ceba 100644 --- a/swh/web/ui/tests/test_service.py +++ b/swh/web/ui/tests/test_service.py @@ -1,155 +1,177 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from nose.tools import istest from unittest.mock import MagicMock, patch from swh.core.hashutil import hex_to_hash from swh.web.ui import service from swh.web.ui.tests import test_app class ServiceTestCase(unittest.TestCase): @classmethod def setUpClass(cls): _, _, cls.storage = test_app.init_app() @istest def lookup_hash_does_not_exist(self): # given self.storage.content_exist = MagicMock(return_value=False) # when actual_lookup = service.lookup_hash( 'sha1:123caf10e9535160d90e874b45aa426de762f19f') # then self.assertFalse(actual_lookup) # check the function has been called with parameters self.storage.content_exist.assert_called_with({ 'sha1': hex_to_hash('123caf10e9535160d90e874b45aa426de762f19f')}) @istest def lookup_hash_exist(self): # given self.storage.content_exist = MagicMock(return_value=True) # when actual_lookup = service.lookup_hash( 'sha1:456caf10e9535160d90e874b45aa426de762f19f') # then self.assertTrue(actual_lookup) self.storage.content_exist.assert_called_with({ 'sha1': hex_to_hash('456caf10e9535160d90e874b45aa426de762f19f')}) @istest def lookup_hash_origin(self): # given self.storage.content_find_occurrence = MagicMock(return_value={ 'origin_type': 'sftp', 'origin_url': 'sftp://ftp.gnu.org/gnu/octave', 'branch': 'octavio-3.4.0.tar.gz', 'revision': b'\xb0L\xaf\x10\xe9SQ`\xd9\x0e\x87KE\xaaBm\xe7b\xf1\x9f', # noqa 'path': b'octavio-3.4.0/doc/interpreter/octave.html/doc_002dS_005fISREG.html' # noqa }) expected_origin = { 'origin_type': 'sftp', 'origin_url': 'sftp://ftp.gnu.org/gnu/octave', 'branch': 'octavio-3.4.0.tar.gz', 'revision': 'b04caf10e9535160d90e874b45aa426de762f19f', 'path': 'octavio-3.4.0/doc/interpreter/octave.html/doc' '_002dS_005fISREG.html' } # when actual_origin = service.lookup_hash_origin( 'sha1_git:456caf10e9535160d90e874b45aa426de762f19f') # then self.assertEqual(actual_origin, expected_origin) self.storage.content_find_occurrence.assert_called_with( {'sha1_git': hex_to_hash('456caf10e9535160d90e874b45aa426de762f19f')}) @istest def stat_counters(self): # given input_stats = { "content": 1770830, "directory": 211683, "directory_entry_dir": 209167, "directory_entry_file": 1807094, "directory_entry_rev": 0, "entity": 0, "entity_history": 0, "occurrence": 0, "occurrence_history": 19600, "origin": 1096, "person": 0, "release": 8584, "revision": 7792, "revision_history": 0, "skipped_content": 0 } self.storage.stat_counters = MagicMock(return_value=input_stats) # when actual_stats = service.stat_counters() # then expected_stats = input_stats self.assertEqual(actual_stats, expected_stats) self.storage.stat_counters.assert_called_with() @istest def hash_and_search(self): # given self.storage.content_exist = MagicMock(return_value=False) bhash = hex_to_hash('456caf10e9535160d90e874b45aa426de762f19f') # when with patch( 'swh.core.hashutil.hashfile', return_value={'sha1': bhash}): actual_hash, actual_search = service.hash_and_search('/some/path') # then self.assertEqual(actual_hash, '456caf10e9535160d90e874b45aa426de762f19f') self.assertFalse(actual_search) self.storage.content_exist.assert_called_with({'sha1': bhash}) @patch('swh.web.ui.service.upload') @istest def test_upload_and_search_upload_OK_basic_case(self, mock_upload): # given (cf. decorators patch) mock_upload.save_in_upload_folder.return_value = ( '/tmp/blah', 'some-filename', None) mock_upload.cleanup.return_value = None file = MagicMock() file.filename = 'some-filename' # when actual_file, actual_hash, actual_search = service.upload_and_search( file) # then self.assertEqual(actual_file, 'some-filename') self.assertIsNone(actual_hash) self.assertIsNone(actual_search) mock_upload.save_in_upload_folder.assert_called_with(file) mock_upload.cleanup.assert_called_with('/tmp/blah') + + @istest + def lookup_origin(self): + # given + self.storage.origin_get = MagicMock(return_value={ + 'id': 'origin-id', + 'lister': 'uuid-lister', + 'project': 'uuid-project', + 'url': 'ftp://some/url/to/origin', + 'type': 'ftp'}) + + # when + actual_origin = service.lookup_origin('origin-id') + + # then + self.assertEqual(actual_origin, {'id': 'origin-id', + 'lister': 'uuid-lister', + 'project': 'uuid-project', + 'url': 'ftp://some/url/to/origin', + 'type': 'ftp'}) + + self.storage.origin_get.assert_called_with({'id': 'origin-id'})