diff --git a/swh/web/ui/service.py b/swh/web/ui/service.py --- a/swh/web/ui/service.py +++ b/swh/web/ui/service.py @@ -74,8 +74,8 @@ Args: query string of the form - Returns: Dict with key found to True or False, according to - whether the checksum is present or not + Returns: Dict with key found containing the hash info if the + hash is present, None if not. """ algo, hash = query.parse_hash(q) @@ -84,6 +84,20 @@ 'algo': algo} +def search_hash(q): + """Checks if the storage contains a given content checksum + + Args: query string of the form + + Returns: Dict with key found to True or False, according to + whether the checksum is present or not + + """ + algo, hash = query.parse_hash(q) + found = backend.content_find(algo, hash) + return {'found': found is not None} + + def lookup_hash_origin(q): """Return information about the checksum contained in the query q. diff --git a/swh/web/ui/templates/upload_and_search.html b/swh/web/ui/templates/upload_and_search.html --- a/swh/web/ui/templates/upload_and_search.html +++ b/swh/web/ui/templates/upload_and_search.html @@ -55,21 +55,25 @@ {{ search_stats['nbfiles'] }} files. {% endif %} - {% if responses is not none and responses %} + {% if search_res is not none %} - {% for resp in responses %} + {% for res in search_res %} - - {% if resp['found'] %} - + {% if res['filename'] is not none %} + + {% else %} + + {% endif %} + {% if res['found'] %} + {% else %} - + {% endif %} @@ -91,8 +95,8 @@ $("#fileinput").inputclick($("#fileElem")); // Make the dropbox available for drag & drop $("#fileElem").filedialog($("#filelist"), $("#searchForm")); - $("#fileinput").filedrop($("#filelist"), $("#searchForm")); // Make the submission button receptive to files + $("#fileinput").filedrop($("#filelist"), $("#searchForm")); {% endblock %} diff --git a/swh/web/ui/tests/test_service.py b/swh/web/ui/tests/test_service.py --- a/swh/web/ui/tests/test_service.py +++ b/swh/web/ui/tests/test_service.py @@ -107,6 +107,45 @@ @patch('swh.web.ui.service.backend') @istest + def search_hash_does_not_exist(self, mock_backend): + # given + mock_backend.content_find = MagicMock(return_value=None) + + # when + actual_lookup = service.search_hash( + 'sha1_git:123caf10e9535160d90e874b45aa426de762f19f') + + # then + self.assertEquals({'found': False}, actual_lookup) + + # check the function has been called with parameters + mock_backend.content_find.assert_called_with( + 'sha1_git', + hex_to_hash('123caf10e9535160d90e874b45aa426de762f19f')) + + @patch('swh.web.ui.service.backend') + @istest + def search_hash_exist(self, mock_backend): + # given + stub_content = { + 'sha1': hex_to_hash('456caf10e9535160d90e874b45aa426de762f19f') + } + mock_backend.content_find = MagicMock(return_value=stub_content) + + # when + actual_lookup = service.search_hash( + 'sha1:456caf10e9535160d90e874b45aa426de762f19f') + + # then + self.assertEquals({'found': True}, actual_lookup) + + mock_backend.content_find.assert_called_with( + 'sha1', + hex_to_hash('456caf10e9535160d90e874b45aa426de762f19f'), + ) + + @patch('swh.web.ui.service.backend') + @istest def lookup_hash_origin(self, mock_backend): # given mock_backend.content_find_occurrence = MagicMock(return_value={ diff --git a/swh/web/ui/tests/views/test_api.py b/swh/web/ui/tests/views/test_api.py --- a/swh/web/ui/tests/views/test_api.py +++ b/swh/web/ui/tests/views/test_api.py @@ -18,6 +18,7 @@ class ApiTestCase(test_app.SWHApiTestCase): + @istest def generic_api_lookup_nothing_is_found(self): # given @@ -287,10 +288,13 @@ @istest def api_search(self, mock_service): # given - mock_service.lookup_hash.return_value = { - 'found': { - 'sha1': 'or something' - } + mock_service.search_hash.return_value = {'found': True} + + expected_result = { + 'search_stats': {'nbfiles': 1, 'pct': 100}, + 'search_res': [{'filename': None, + 'sha1': 'sha1:blah', + 'found': True}] } # when @@ -298,19 +302,21 @@ self.assertEquals(rv.status_code, 200) self.assertEquals(rv.mimetype, 'application/json') - response_data = json.loads(rv.data.decode('utf-8')) - self.assertEquals(response_data, {'found': True}) - mock_service.lookup_hash.assert_called_once_with('sha1:blah') + response_data = json.loads(rv.data.decode('utf-8')) + self.assertEquals(response_data, expected_result) + mock_service.search_hash.assert_called_once_with('sha1:blah') @patch('swh.web.ui.views.api.service') @istest def api_search_as_yaml(self, mock_service): # given - mock_service.lookup_hash.return_value = { - 'found': { - 'sha1': 'sha1 hash' - } + mock_service.search_hash.return_value = {'found': True} + expected_result = { + 'search_stats': {'nbfiles': 1, 'pct': 100}, + 'search_res': [{'filename': None, + 'sha1': 'sha1:halb', + 'found': True}] } # when @@ -321,15 +327,22 @@ self.assertEquals(rv.mimetype, 'application/yaml') response_data = yaml.load(rv.data.decode('utf-8')) - self.assertEquals(response_data, {'found': True}) + self.assertEquals(response_data, expected_result) - mock_service.lookup_hash.assert_called_once_with('sha1:halb') + mock_service.search_hash.assert_called_once_with('sha1:halb') @patch('swh.web.ui.views.api.service') @istest def api_search_not_found(self, mock_service): # given - mock_service.lookup_hash.return_value = {} + mock_service.search_hash.return_value = {'found': False} + + expected_result = { + 'search_stats': {'nbfiles': 1, 'pct': 0}, + 'search_res': [{'filename': None, + 'sha1': 'sha1:halb', + 'found': False}] + } # when rv = self.app.get('/api/1/search/sha1:halb/') @@ -337,9 +350,9 @@ self.assertEquals(rv.status_code, 200) self.assertEquals(rv.mimetype, 'application/json') response_data = json.loads(rv.data.decode('utf-8')) - self.assertEquals(response_data, {'found': False}) + self.assertEquals(response_data, expected_result) - mock_service.lookup_hash.assert_called_once_with('sha1:halb') + mock_service.search_hash.assert_called_once_with('sha1:halb') @patch('swh.web.ui.views.api.service') @istest diff --git a/swh/web/ui/tests/views/test_browse.py b/swh/web/ui/tests/views/test_browse.py --- a/swh/web/ui/tests/views/test_browse.py +++ b/swh/web/ui/tests/views/test_browse.py @@ -26,78 +26,84 @@ rv = self.client.get('/search/') self.assertEqual(rv.status_code, 200) - self.assertEqual(self.get_context_variable('messages'), []) - self.assertEqual(self.get_context_variable('responses'), []) + self.assertEqual(self.get_context_variable('message'), '') + self.assertEqual(self.get_context_variable('search_res'), None) self.assert_template_used('upload_and_search.html') - @patch('swh.web.ui.views.browse.service') + @patch('swh.web.ui.views.browse.api') @istest - def search_get_query_hash_not_found(self, mock_service): + def search_get_query_hash_not_found(self, mock_api): # given - mock_service.lookup_hash.return_value = {'found': None} + mock_api.api_search.return_value = { + 'search_res': [{ + 'filename': None, + 'sha1': 'sha1:456', + 'found': False}], + 'search_stats': {'nbfiles': 1, 'pct': 100}} # when rv = self.client.get('/search/?q=sha1:456') self.assertEqual(rv.status_code, 200) - self.assertEqual(self.get_context_variable('q'), 'sha1:456') - self.assertEqual(self.get_context_variable('messages'), []) - self.assertEqual(self.get_context_variable('responses'), [ - {'filename': 'User submitted hash', + self.assertEqual(self.get_context_variable('message'), '') + self.assertEqual(self.get_context_variable('search_res'), [ + {'filename': None, 'sha1': 'sha1:456', 'found': False}]) self.assert_template_used('upload_and_search.html') - mock_service.lookup_hash.assert_called_once_with('sha1:456') + mock_api.api_search.assert_called_once_with('sha1:456') - @patch('swh.web.ui.views.browse.service') + @patch('swh.web.ui.views.browse.api') @istest - def search_get_query_hash_bad_input(self, mock_service): + def search_get_query_hash_bad_input(self, mock_api): # given - mock_service.lookup_hash.side_effect = BadInputExc('error msg') + mock_api.api_search.side_effect = BadInputExc('error msg') # when rv = self.client.get('/search/?q=sha1_git:789') self.assertEqual(rv.status_code, 200) - self.assertEqual(self.get_context_variable('q'), 'sha1_git:789') - self.assertEqual(self.get_context_variable('messages'), ['error msg']) - self.assertEqual(self.get_context_variable('responses'), []) + self.assertEqual(self.get_context_variable('message'), 'error msg') + self.assertEqual(self.get_context_variable('search_res'), None) self.assert_template_used('upload_and_search.html') - mock_service.lookup_hash.assert_called_once_with('sha1_git:789') + mock_api.api_search.assert_called_once_with('sha1_git:789') - @patch('swh.web.ui.views.browse.service') + @patch('swh.web.ui.views.browse.api') @istest - def search_get_query_hash_found(self, mock_service): + def search_get_query_hash_found(self, mock_api): # given - mock_service.lookup_hash.return_value = {'found': True} + mock_api.api_search.return_value = { + 'search_res': [{ + 'filename': None, + 'sha1': 'sha1:123', + 'found': True}], + 'search_stats': {'nbfiles': 1, 'pct': 100}} # when rv = self.client.get('/search/?q=sha1:123') self.assertEqual(rv.status_code, 200) - self.assertEqual(self.get_context_variable('q'), 'sha1:123') - self.assertEqual(self.get_context_variable('messages'), []) - self.assertEqual(len(self.get_context_variable('responses')), 1) - resp = self.get_context_variable('responses')[0] + self.assertEqual(self.get_context_variable('message'), '') + self.assertEqual(len(self.get_context_variable('search_res')), 1) + resp = self.get_context_variable('search_res')[0] self.assertTrue(resp is not None) self.assertEqual(resp['sha1'], 'sha1:123') self.assertEqual(resp['found'], True) self.assert_template_used('upload_and_search.html') - mock_service.lookup_hash.assert_called_once_with('sha1:123') + mock_api.api_search.assert_called_once_with('sha1:123') - @patch('swh.web.ui.views.browse.service') @patch('swh.web.ui.views.browse.request') + @patch('swh.web.ui.views.browse.api') @istest - def search_post_hashes_bad_input(self, mock_request, - mock_service): + def search_post_hashes_bad_input(self, mock_api, mock_request): # given mock_request.form = {'a': ['456caf10e9535160d90e874b45aa426de762f19f'], 'b': ['745bab676c8f3cec8016e0c39ea61cf57e518865']} mock_request.method = 'POST' - mock_service.lookup_multiple_hashes.side_effect = BadInputExc( + mock_api.api_search.side_effect = BadInputExc( 'error bad input') # when (mock_request completes the post request) @@ -107,83 +113,84 @@ self.assertEqual(rv.status_code, 200) self.assertEqual(self.get_context_variable('search_stats'), {'nbfiles': 0, 'pct': 0}) - self.assertEqual(self.get_context_variable('responses'), []) - self.assertEqual(self.get_context_variable('messages'), - ['error bad input']) + self.assertEqual(self.get_context_variable('search_res'), None) + self.assertEqual(self.get_context_variable('message'), + 'error bad input') self.assert_template_used('upload_and_search.html') - mock_service.upload_and_search.called = True - - @patch('swh.web.ui.views.browse.service') @patch('swh.web.ui.views.browse.request') + @patch('swh.web.ui.views.browse.api') @istest - def search_post_hashes_none(self, mock_request, mock_service): + def search_post_hashes_none(self, mock_api, mock_request): # given mock_request.form = {'a': ['456caf10e9535160d90e874b45aa426de762f19f'], 'b': ['745bab676c8f3cec8016e0c39ea61cf57e518865']} mock_request.method = 'POST' - mock_service.lookup_multiple_hashes.return_value = [ - {'filename': 'a', - 'sha1': '456caf10e9535160d90e874b45aa426de762f19f', - 'found': False}, - {'filename': 'b', - 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865', - 'found': False} - ] + mock_api.api_search.return_value = { + 'search_stats': {'nbfiles': 2, 'pct': 0}, + 'search_res': [{'filename': 'a', + 'sha1': '456caf10e9535160d90e874b45aa426de762f19f', + 'found': False}, + {'filename': 'b', + 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865', + 'found': False}]} # when (mock_request completes the post request) rv = self.client.post('/search/') # then self.assertEqual(rv.status_code, 200) - self.assertEqual(len(self.get_context_variable('responses')), 2) + self.assertIsNotNone(self.get_context_variable('search_res')) self.assertTrue(self.get_context_variable('search_stats') is not None) + self.assertEqual(len(self.get_context_variable('search_res')), 2) + stats = self.get_context_variable('search_stats') self.assertEqual(stats['nbfiles'], 2) self.assertEqual(stats['pct'], 0) - a, b = self.get_context_variable('responses') + + a, b = self.get_context_variable('search_res') self.assertEqual(a['found'], False) self.assertEqual(b['found'], False) - self.assertEqual(self.get_context_variable('messages'), []) - self.assert_template_used('upload_and_search.html') + self.assertEqual(self.get_context_variable('message'), '') - mock_service.upload_and_search.called = True + self.assert_template_used('upload_and_search.html') - @patch('swh.web.ui.views.browse.service') @patch('swh.web.ui.views.browse.request') + @patch('swh.web.ui.views.browse.api') @istest - def search_post_hashes_some(self, mock_request, mock_service): + def search_post_hashes_some(self, mock_api, mock_request): # given mock_request.form = {'a': '456caf10e9535160d90e874b45aa426de762f19f', 'b': '745bab676c8f3cec8016e0c39ea61cf57e518865'} mock_request.method = 'POST' - mock_service.lookup_multiple_hashes.return_value = [ - {'filename': 'a', - 'sha1': '456caf10e9535160d90e874b45aa426de762f19f', - 'found': False}, - {'filename': 'b', - 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865', - 'found': True} - ] + mock_api.api_search.return_value = { + 'search_stats': {'nbfiles': 2, 'pct': 50}, + 'search_res': [{'filename': 'a', + 'sha1': '456caf10e9535160d90e874b45aa426de762f19f', + 'found': False}, + {'filename': 'b', + 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865', + 'found': True}]} # when (mock_request completes the post request) rv = self.client.post('/search/') # then self.assertEqual(rv.status_code, 200) - self.assertEqual(len(self.get_context_variable('responses')), 2) + self.assertIsNotNone(self.get_context_variable('search_res')) + self.assertEqual(len(self.get_context_variable('search_res')), 2) self.assertTrue(self.get_context_variable('search_stats') is not None) + stats = self.get_context_variable('search_stats') self.assertEqual(stats['nbfiles'], 2) self.assertEqual(stats['pct'], 50) - self.assertEqual(self.get_context_variable('messages'), []) - a, b = self.get_context_variable('responses') + self.assertEqual(self.get_context_variable('message'), '') + + a, b = self.get_context_variable('search_res') self.assertEqual(a['found'], False) self.assertEqual(b['found'], True) self.assert_template_used('upload_and_search.html') - mock_service.upload_and_search.called = True - class ContentView(test_app.SWHViewTestCase): render_template = False diff --git a/swh/web/ui/views/api.py b/swh/web/ui/views/api.py --- a/swh/web/ui/views/api.py +++ b/swh/web/ui/views/api.py @@ -23,7 +23,7 @@ return service.stat_counters() -@app.route('/api/1/search/') +@app.route('/api/1/search/', methods=['POST']) @app.route('/api/1/search//') def api_search(q): """Search a content per hash. @@ -42,8 +42,48 @@ GET /api/1/search/sha1:bd819b5b28fcde3bf114d16a44ac46250da94ee5/ """ - r = service.lookup_hash(q).get('found') - return {'found': True if r else False} + + response = {'search_res': None, + 'search_stats': None} + search_stats = {'nbfiles': 0, 'pct': 0} + search_res = None + + # Single hash request route + if q: + r = service.search_hash(q) + search_res = [{'filename': None, + 'sha1': q, + 'found': r['found']}] + search_stats['nbfiles'] = 1 + search_stats['pct'] = 100 if r['found'] else 0 + + # Post form submission with many hash requests + elif request.method == 'POST': + data = request.form + queries = [] + # Remove potential inputs with no associated value + for k, v in data.items(): + if v is not None: + if k == 'q': + queries.append({'filename': None, 'sha1': v}) + elif v != '': + queries.append({'filename': k, 'sha1': v}) + + if len(queries) > 0: + lookup = service.lookup_multiple_hashes(queries) + result = [] + for el in lookup: + result.append({'filename': el['filename'], + 'sha1': el['sha1'], + 'found': el['found']}) + search_res = result + nbfound = len([x for x in lookup if x['found']]) + search_stats['nbfiles'] = len(queries) + search_stats['pct'] = (nbfound / len(queries))*100 + + response['search_res'] = search_res + response['search_stats'] = search_stats + return response def _api_lookup(criteria, diff --git a/swh/web/ui/views/browse.py b/swh/web/ui/views/browse.py --- a/swh/web/ui/views/browse.py +++ b/swh/web/ui/views/browse.py @@ -38,55 +38,39 @@ TODO: Batch-process with all checksums, not just sha1 """ - env = {'q': None, + + env = {'search_res': None, 'search_stats': None, - 'responses': None, - 'messages': []} + 'message': []} - search_stats = None - responses = [] - messages = [] + search_stats = {'nbfiles': 0, 'pct': 0} + search_res = None + message = '' # Get with a single hash request if request.method == 'GET': data = request.args q = data.get('q') - env['q'] = q if q: try: - search_stats = {'nbfiles': 0, 'pct': 0} - r = service.lookup_hash(q) - responses.append({'filename': 'User submitted hash', - 'sha1': q, - 'found': r.get('found') is not None}) - search_stats['nbfiles'] = 1 - search_stats['pct'] = 100 if r.get('found') is not None else 0 + search = api.api_search(q) + search_res = search['search_res'] + search_stats = search['search_stats'] except BadInputExc as e: - messages.append(str(e)) + message = str(e) - # POST form submission with many hash requests + # Post form submission with many hash requests elif request.method == 'POST': - data = request.form - search_stats = {'nbfiles': 0, 'pct': 0} - queries = [] - # Remove potential inputs with no associated value - for k, v in data.items(): - if v is not None and v != '': - queries.append({'filename': k, 'sha1': v}) - - if len(queries) > 0: - try: - lookup = service.lookup_multiple_hashes(queries) - nbfound = len([x for x in lookup if x['found']]) - responses = lookup - search_stats['nbfiles'] = len(queries) - search_stats['pct'] = (nbfound / len(queries))*100 - except BadInputExc as e: - messages.append(str(e)) + try: + search = api.api_search(None) + search_res = search['search_res'] + search_stats = search['search_stats'] + except BadInputExc as e: + message = str(e) env['search_stats'] = search_stats - env['responses'] = responses - env['messages'] = messages + env['search_res'] = search_res + env['message'] = message return render_template('upload_and_search.html', **env)
File name SHA1 checksum Result
{{ resp['filename'] }}{{ resp['sha1'] }}{{ res['filename'] }}From text input{{ res['sha1'] }} {{ resp['sha1'] }}{{ res['sha1'] }}