D47.id157.diff
No OneTemporary
Actions

Size

23 KB

Subscribers

None

D47.id157.diff
View Options

	diff --git a/swh/web/ui/service.py b/swh/web/ui/service.py
	--- a/swh/web/ui/service.py
	+++ b/swh/web/ui/service.py
	@@ -74,8 +74,8 @@

	Args: query string of the form <hash_algo:hash>

	- Returns: Dict with key found to True or False, according to
	- whether the checksum is present or not
	+ Returns: Dict with key found containing the hash info if the
	+ hash is present, None if not.

	"""
	algo, hash = query.parse_hash(q)
	@@ -84,6 +84,20 @@
	'algo': algo}


	+def search_hash(q):
	+ """Checks if the storage contains a given content checksum
	+
	+ Args: query string of the form <hash_algo:hash>
	+
	+ Returns: Dict with key found to True or False, according to
	+ whether the checksum is present or not
	+
	+ """
	+ algo, hash = query.parse_hash(q)
	+ found = backend.content_find(algo, hash)
	+ return {'found': found is not None}
	+
	+
	def lookup_hash_origin(q):
	"""Return information about the checksum contained in the query q.

	diff --git a/swh/web/ui/templates/upload_and_search.html b/swh/web/ui/templates/upload_and_search.html
	--- a/swh/web/ui/templates/upload_and_search.html
	+++ b/swh/web/ui/templates/upload_and_search.html
	@@ -55,21 +55,25 @@
	{{ search_stats['nbfiles'] }} files.
	</label>
	{% endif %}
	- {% if responses is not none and responses %}
	+ {% if search_res is not none %}
	<table id="results" class="table table-striped">
	<thead class="thead-default">
	<th>File name</th>
	<th>SHA1 checksum</th>
	<th>Result</th>
	</thead>
	- {% for resp in responses %}
	+ {% for res in search_res %}
	<tr>
	- <td>{{ resp['filename'] }}</td>
	- {% if resp['found'] %}
	- <td><a href="{{ url_for('browse_content') }}sha1:{{ resp['sha1'] }}">{{ resp['sha1'] }}</a></td>
	+ {% if res['filename'] is not none %}
	+ <td>{{ res['filename'] }}</td>
	+ {% else %}
	+ <td>From text input</td>
	+ {% endif %}
	+ {% if res['found'] %}
	+ <td><a href="{{ url_for('browse_content') }}sha1:{{ res['sha1'] }}">{{ res['sha1'] }}</a></td>
	<td><span class="glyphicon glyphicon-ok file-found"></span></td>
	{% else %}
	- <td>{{ resp['sha1'] }}</td>
	+ <td>{{ res['sha1'] }}</td>
	<td><span class="glyphicon glyphicon-remove file-notfound"></span></td>
	{% endif %}
	</tr>
	@@ -91,8 +95,8 @@
	$("#fileinput").inputclick($("#fileElem"));
	// Make the dropbox available for drag & drop
	$("#fileElem").filedialog($("#filelist"), $("#searchForm"));
	- $("#fileinput").filedrop($("#filelist"), $("#searchForm"));
	// Make the submission button receptive to files
	+ $("#fileinput").filedrop($("#filelist"), $("#searchForm"));
	</script>
	</div>
	{% endblock %}
	diff --git a/swh/web/ui/tests/test_service.py b/swh/web/ui/tests/test_service.py
	--- a/swh/web/ui/tests/test_service.py
	+++ b/swh/web/ui/tests/test_service.py
	@@ -107,6 +107,45 @@

	@patch('swh.web.ui.service.backend')
	@istest
	+ def search_hash_does_not_exist(self, mock_backend):
	+ # given
	+ mock_backend.content_find = MagicMock(return_value=None)
	+
	+ # when
	+ actual_lookup = service.search_hash(
	+ 'sha1_git:123caf10e9535160d90e874b45aa426de762f19f')
	+
	+ # then
	+ self.assertEquals({'found': False}, actual_lookup)
	+
	+ # check the function has been called with parameters
	+ mock_backend.content_find.assert_called_with(
	+ 'sha1_git',
	+ hex_to_hash('123caf10e9535160d90e874b45aa426de762f19f'))
	+
	+ @patch('swh.web.ui.service.backend')
	+ @istest
	+ def search_hash_exist(self, mock_backend):
	+ # given
	+ stub_content = {
	+ 'sha1': hex_to_hash('456caf10e9535160d90e874b45aa426de762f19f')
	+ }
	+ mock_backend.content_find = MagicMock(return_value=stub_content)
	+
	+ # when
	+ actual_lookup = service.search_hash(
	+ 'sha1:456caf10e9535160d90e874b45aa426de762f19f')
	+
	+ # then
	+ self.assertEquals({'found': True}, actual_lookup)
	+
	+ mock_backend.content_find.assert_called_with(
	+ 'sha1',
	+ hex_to_hash('456caf10e9535160d90e874b45aa426de762f19f'),
	+ )
	+
	+ @patch('swh.web.ui.service.backend')
	+ @istest
	def lookup_hash_origin(self, mock_backend):
	# given
	mock_backend.content_find_occurrence = MagicMock(return_value={
	diff --git a/swh/web/ui/tests/views/test_api.py b/swh/web/ui/tests/views/test_api.py
	--- a/swh/web/ui/tests/views/test_api.py
	+++ b/swh/web/ui/tests/views/test_api.py
	@@ -18,6 +18,7 @@


	class ApiTestCase(test_app.SWHApiTestCase):
	+
	@istest
	def generic_api_lookup_nothing_is_found(self):
	# given
	@@ -287,10 +288,13 @@
	@istest
	def api_search(self, mock_service):
	# given
	- mock_service.lookup_hash.return_value = {
	- 'found': {
	- 'sha1': 'or something'
	- }
	+ mock_service.search_hash.return_value = {'found': True}
	+
	+ expected_result = {
	+ 'search_stats': {'nbfiles': 1, 'pct': 100},
	+ 'search_res': [{'filename': None,
	+ 'sha1': 'sha1:blah',
	+ 'found': True}]
	}

	# when
	@@ -298,19 +302,21 @@

	self.assertEquals(rv.status_code, 200)
	self.assertEquals(rv.mimetype, 'application/json')
	- response_data = json.loads(rv.data.decode('utf-8'))
	- self.assertEquals(response_data, {'found': True})

	- mock_service.lookup_hash.assert_called_once_with('sha1:blah')
	+ response_data = json.loads(rv.data.decode('utf-8'))
	+ self.assertEquals(response_data, expected_result)
	+ mock_service.search_hash.assert_called_once_with('sha1:blah')

	@patch('swh.web.ui.views.api.service')
	@istest
	def api_search_as_yaml(self, mock_service):
	# given
	- mock_service.lookup_hash.return_value = {
	- 'found': {
	- 'sha1': 'sha1 hash'
	- }
	+ mock_service.search_hash.return_value = {'found': True}
	+ expected_result = {
	+ 'search_stats': {'nbfiles': 1, 'pct': 100},
	+ 'search_res': [{'filename': None,
	+ 'sha1': 'sha1:halb',
	+ 'found': True}]
	}

	# when
	@@ -321,15 +327,23 @@
	self.assertEquals(rv.mimetype, 'application/yaml')

	response_data = yaml.load(rv.data.decode('utf-8'))
	- self.assertEquals(response_data, {'found': True})
	+ self.assertEquals(response_data, expected_result)

	- mock_service.lookup_hash.assert_called_once_with('sha1:halb')
	+ mock_service.search_hash.assert_called_once_with('sha1:halb')

	@patch('swh.web.ui.views.api.service')
	@istest
	def api_search_not_found(self, mock_service):
	# given
	- mock_service.lookup_hash.return_value = {}
	+ mock_service.search_hash.return_value = {'found': False,
	+ 'algo': 'sha1'}
	+
	+ expected_result = {
	+ 'search_stats': {'nbfiles': 1, 'pct': 0},
	+ 'search_res': [{'filename': None,
	+ 'sha1': 'sha1:halb',
	+ 'found': False}]
	+ }

	# when
	rv = self.app.get('/api/1/search/sha1:halb/')
	@@ -337,9 +351,9 @@
	self.assertEquals(rv.status_code, 200)
	self.assertEquals(rv.mimetype, 'application/json')
	response_data = json.loads(rv.data.decode('utf-8'))
	- self.assertEquals(response_data, {'found': False})
	+ self.assertEquals(response_data, expected_result)

	- mock_service.lookup_hash.assert_called_once_with('sha1:halb')
	+ mock_service.search_hash.assert_called_once_with('sha1:halb')

	@patch('swh.web.ui.views.api.service')
	@istest
	diff --git a/swh/web/ui/tests/views/test_browse.py b/swh/web/ui/tests/views/test_browse.py
	--- a/swh/web/ui/tests/views/test_browse.py
	+++ b/swh/web/ui/tests/views/test_browse.py
	@@ -26,78 +26,84 @@
	rv = self.client.get('/search/')

	self.assertEqual(rv.status_code, 200)
	- self.assertEqual(self.get_context_variable('messages'), [])
	- self.assertEqual(self.get_context_variable('responses'), [])
	+ self.assertEqual(self.get_context_variable('message'), '')
	+ self.assertEqual(self.get_context_variable('search_res'), None)
	self.assert_template_used('upload_and_search.html')

	- @patch('swh.web.ui.views.browse.service')
	+ @patch('swh.web.ui.views.browse.api')
	@istest
	- def search_get_query_hash_not_found(self, mock_service):
	+ def search_get_query_hash_not_found(self, mock_api):
	# given
	- mock_service.lookup_hash.return_value = {'found': None}
	+ mock_api.api_search.return_value = {
	+ 'search_res': [{
	+ 'filename': None,
	+ 'sha1': 'sha1:456',
	+ 'found': False}],
	+ 'search_stats': {'nbfiles': 1, 'pct': 100}}

	# when
	rv = self.client.get('/search/?q=sha1:456')

	self.assertEqual(rv.status_code, 200)
	- self.assertEqual(self.get_context_variable('q'), 'sha1:456')
	- self.assertEqual(self.get_context_variable('messages'), [])
	- self.assertEqual(self.get_context_variable('responses'), [
	- {'filename': 'User submitted hash',
	+ self.assertEqual(self.get_context_variable('message'), '')
	+ self.assertEqual(self.get_context_variable('search_res'), [
	+ {'filename': None,
	'sha1': 'sha1:456',
	'found': False}])
	self.assert_template_used('upload_and_search.html')

	- mock_service.lookup_hash.assert_called_once_with('sha1:456')
	+ mock_api.api_search.assert_called_once_with('sha1:456')

	- @patch('swh.web.ui.views.browse.service')
	+ @patch('swh.web.ui.views.browse.api')
	@istest
	- def search_get_query_hash_bad_input(self, mock_service):
	+ def search_get_query_hash_bad_input(self, mock_api):
	# given
	- mock_service.lookup_hash.side_effect = BadInputExc('error msg')
	+ mock_api.api_search.side_effect = BadInputExc('error msg')

	# when
	rv = self.client.get('/search/?q=sha1_git:789')

	self.assertEqual(rv.status_code, 200)
	- self.assertEqual(self.get_context_variable('q'), 'sha1_git:789')
	- self.assertEqual(self.get_context_variable('messages'), ['error msg'])
	- self.assertEqual(self.get_context_variable('responses'), [])
	+ self.assertEqual(self.get_context_variable('message'), 'error msg')
	+ self.assertEqual(self.get_context_variable('search_res'), None)
	self.assert_template_used('upload_and_search.html')

	- mock_service.lookup_hash.assert_called_once_with('sha1_git:789')
	+ mock_api.api_search.assert_called_once_with('sha1_git:789')

	- @patch('swh.web.ui.views.browse.service')
	+ @patch('swh.web.ui.views.browse.api')
	@istest
	- def search_get_query_hash_found(self, mock_service):
	+ def search_get_query_hash_found(self, mock_api):
	# given
	- mock_service.lookup_hash.return_value = {'found': True}
	+ mock_api.api_search.return_value = {
	+ 'search_res': [{
	+ 'filename': None,
	+ 'sha1': 'sha1:123',
	+ 'found': True}],
	+ 'search_stats': {'nbfiles': 1, 'pct': 100}}

	# when
	rv = self.client.get('/search/?q=sha1:123')

	self.assertEqual(rv.status_code, 200)
	- self.assertEqual(self.get_context_variable('q'), 'sha1:123')
	- self.assertEqual(self.get_context_variable('messages'), [])
	- self.assertEqual(len(self.get_context_variable('responses')), 1)
	- resp = self.get_context_variable('responses')[0]
	+ self.assertEqual(self.get_context_variable('message'), '')
	+ self.assertEqual(len(self.get_context_variable('search_res')), 1)
	+ resp = self.get_context_variable('search_res')[0]
	self.assertTrue(resp is not None)
	self.assertEqual(resp['sha1'], 'sha1:123')
	self.assertEqual(resp['found'], True)
	self.assert_template_used('upload_and_search.html')

	- mock_service.lookup_hash.assert_called_once_with('sha1:123')
	+ mock_api.api_search.assert_called_once_with('sha1:123')

	- @patch('swh.web.ui.views.browse.service')
	@patch('swh.web.ui.views.browse.request')
	+ @patch('swh.web.ui.views.browse.api')
	@istest
	- def search_post_hashes_bad_input(self, mock_request,
	- mock_service):
	+ def search_post_hashes_bad_input(self, mock_api, mock_request):
	# given
	mock_request.form = {'a': ['456caf10e9535160d90e874b45aa426de762f19f'],
	'b': ['745bab676c8f3cec8016e0c39ea61cf57e518865']}
	mock_request.method = 'POST'
	- mock_service.lookup_multiple_hashes.side_effect = BadInputExc(
	+ mock_api.api_search.side_effect = BadInputExc(
	'error bad input')

	# when (mock_request completes the post request)
	@@ -107,83 +113,84 @@
	self.assertEqual(rv.status_code, 200)
	self.assertEqual(self.get_context_variable('search_stats'),
	{'nbfiles': 0, 'pct': 0})
	- self.assertEqual(self.get_context_variable('responses'), [])
	- self.assertEqual(self.get_context_variable('messages'),
	- ['error bad input'])
	+ self.assertEqual(self.get_context_variable('search_res'), None)
	+ self.assertEqual(self.get_context_variable('message'),
	+ 'error bad input')
	self.assert_template_used('upload_and_search.html')

	- mock_service.upload_and_search.called = True
	-
	- @patch('swh.web.ui.views.browse.service')
	@patch('swh.web.ui.views.browse.request')
	+ @patch('swh.web.ui.views.browse.api')
	@istest
	- def search_post_hashes_none(self, mock_request, mock_service):
	+ def search_post_hashes_none(self, mock_api, mock_request):
	# given
	mock_request.form = {'a': ['456caf10e9535160d90e874b45aa426de762f19f'],
	'b': ['745bab676c8f3cec8016e0c39ea61cf57e518865']}
	mock_request.method = 'POST'
	- mock_service.lookup_multiple_hashes.return_value = [
	- {'filename': 'a',
	- 'sha1': '456caf10e9535160d90e874b45aa426de762f19f',
	- 'found': False},
	- {'filename': 'b',
	- 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865',
	- 'found': False}
	- ]
	+ mock_api.api_search.return_value = {
	+ 'search_stats': {'nbfiles': 2, 'pct': 0},
	+ 'search_res': [{'filename': 'a',
	+ 'sha1': '456caf10e9535160d90e874b45aa426de762f19f',
	+ 'found': False},
	+ {'filename': 'b',
	+ 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865',
	+ 'found': False}]}

	# when (mock_request completes the post request)
	rv = self.client.post('/search/')

	# then
	self.assertEqual(rv.status_code, 200)
	- self.assertEqual(len(self.get_context_variable('responses')), 2)
	+ self.assertIsNotNone(self.get_context_variable('search_res'))
	self.assertTrue(self.get_context_variable('search_stats') is not None)
	+ self.assertEqual(len(self.get_context_variable('search_res')), 2)
	+
	stats = self.get_context_variable('search_stats')
	self.assertEqual(stats['nbfiles'], 2)
	self.assertEqual(stats['pct'], 0)
	- a, b = self.get_context_variable('responses')
	+
	+ a, b = self.get_context_variable('search_res')
	self.assertEqual(a['found'], False)
	self.assertEqual(b['found'], False)
	- self.assertEqual(self.get_context_variable('messages'), [])
	- self.assert_template_used('upload_and_search.html')
	+ self.assertEqual(self.get_context_variable('message'), '')

	- mock_service.upload_and_search.called = True
	+ self.assert_template_used('upload_and_search.html')

	- @patch('swh.web.ui.views.browse.service')
	@patch('swh.web.ui.views.browse.request')
	+ @patch('swh.web.ui.views.browse.api')
	@istest
	- def search_post_hashes_some(self, mock_request, mock_service):
	+ def search_post_hashes_some(self, mock_api, mock_request):
	# given
	mock_request.form = {'a': '456caf10e9535160d90e874b45aa426de762f19f',
	'b': '745bab676c8f3cec8016e0c39ea61cf57e518865'}
	mock_request.method = 'POST'
	- mock_service.lookup_multiple_hashes.return_value = [
	- {'filename': 'a',
	- 'sha1': '456caf10e9535160d90e874b45aa426de762f19f',
	- 'found': False},
	- {'filename': 'b',
	- 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865',
	- 'found': True}
	- ]
	+ mock_api.api_search.return_value = {
	+ 'search_stats': {'nbfiles': 2, 'pct': 50},
	+ 'search_res': [{'filename': 'a',
	+ 'sha1': '456caf10e9535160d90e874b45aa426de762f19f',
	+ 'found': False},
	+ {'filename': 'b',
	+ 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865',
	+ 'found': True}]}

	# when (mock_request completes the post request)
	rv = self.client.post('/search/')

	# then
	self.assertEqual(rv.status_code, 200)
	- self.assertEqual(len(self.get_context_variable('responses')), 2)
	+ self.assertIsNotNone(self.get_context_variable('search_res'))
	+ self.assertEqual(len(self.get_context_variable('search_res')), 2)
	self.assertTrue(self.get_context_variable('search_stats') is not None)
	+
	stats = self.get_context_variable('search_stats')
	self.assertEqual(stats['nbfiles'], 2)
	self.assertEqual(stats['pct'], 50)
	- self.assertEqual(self.get_context_variable('messages'), [])
	- a, b = self.get_context_variable('responses')
	+ self.assertEqual(self.get_context_variable('message'), '')
	+
	+ a, b = self.get_context_variable('search_res')
	self.assertEqual(a['found'], False)
	self.assertEqual(b['found'], True)
	self.assert_template_used('upload_and_search.html')

	- mock_service.upload_and_search.called = True
	-

	class ContentView(test_app.SWHViewTestCase):
	render_template = False
	diff --git a/swh/web/ui/views/api.py b/swh/web/ui/views/api.py
	--- a/swh/web/ui/views/api.py
	+++ b/swh/web/ui/views/api.py
	@@ -23,7 +23,7 @@
	return service.stat_counters()


	-@app.route('/api/1/search/')
	+@app.route('/api/1/search/', methods=['POST'])
	@app.route('/api/1/search/<string:q>/')
	def api_search(q):
	"""Search a content per hash.
	@@ -42,8 +42,48 @@
	GET /api/1/search/sha1:bd819b5b28fcde3bf114d16a44ac46250da94ee5/

	"""
	- r = service.lookup_hash(q).get('found')
	- return {'found': True if r else False}
	+
	+ response = {'search_res': None,
	+ 'search_stats': None}
	+ search_stats = {'nbfiles': 0, 'pct': 0}
	+ search_res = None
	+
	+ # Single hash request route
	+ if q:
	+ r = service.search_hash(q)
	+ search_res = [{'filename': None,
	+ 'sha1': q,
	+ 'found': r['found']}]
	+ search_stats['nbfiles'] = 1
	+ search_stats['pct'] = 100 if r['found'] else 0
	+
	+ # Post form submission with many hash requests
	+ elif request.method == 'POST':
	+ data = request.form
	+ queries = []
	+ # Remove potential inputs with no associated value
	+ for k, v in data.items():
	+ if v is not None:
	+ if k == 'q':
	+ queries.append({'filename': None, 'sha1': v})
	+ elif v != '':
	+ queries.append({'filename': k, 'sha1': v})
	+
	+ if len(queries) > 0:
	+ lookup = service.lookup_multiple_hashes(queries)
	+ result = []
	+ for el in lookup:
	+ result.append({'filename': el['filename'],
	+ 'sha1': el['sha1'],
	+ 'found': el['found']})
	+ search_res = result
	+ nbfound = len([x for x in lookup if x['found']])
	+ search_stats['nbfiles'] = len(queries)
	+ search_stats['pct'] = (nbfound / len(queries))*100
	+
	+ response['search_res'] = search_res
	+ response['search_stats'] = search_stats
	+ return response


	def _api_lookup(criteria,
	diff --git a/swh/web/ui/views/browse.py b/swh/web/ui/views/browse.py
	--- a/swh/web/ui/views/browse.py
	+++ b/swh/web/ui/views/browse.py
	@@ -38,55 +38,39 @@
	TODO:
	Batch-process with all checksums, not just sha1
	"""
	- env = {'q': None,
	+
	+ env = {'search_res': None,
	'search_stats': None,
	- 'responses': None,
	- 'messages': []}
	+ 'message': []}

	- search_stats = None
	- responses = []
	- messages = []
	+ search_stats = {'nbfiles': 0, 'pct': 0}
	+ search_res = None
	+ message = ''

	# Get with a single hash request
	if request.method == 'GET':
	data = request.args
	q = data.get('q')
	- env['q'] = q
	if q:
	try:
	- search_stats = {'nbfiles': 0, 'pct': 0}
	- r = service.lookup_hash(q)
	- responses.append({'filename': 'User submitted hash',
	- 'sha1': q,
	- 'found': r.get('found') is not None})
	- search_stats['nbfiles'] = 1
	- search_stats['pct'] = 100 if r.get('found') is not None else 0
	+ search = api.api_search(q)
	+ search_res = search['search_res']
	+ search_stats = search['search_stats']
	except BadInputExc as e:
	- messages.append(str(e))
	+ message = str(e)

	- # POST form submission with many hash requests
	+ # Post form submission with many hash requests
	elif request.method == 'POST':
	- data = request.form
	- search_stats = {'nbfiles': 0, 'pct': 0}
	- queries = []
	- # Remove potential inputs with no associated value
	- for k, v in data.items():
	- if v is not None and v != '':
	- queries.append({'filename': k, 'sha1': v})
	-
	- if len(queries) > 0:
	- try:
	- lookup = service.lookup_multiple_hashes(queries)
	- nbfound = len([x for x in lookup if x['found']])
	- responses = lookup
	- search_stats['nbfiles'] = len(queries)
	- search_stats['pct'] = (nbfound / len(queries))*100
	- except BadInputExc as e:
	- messages.append(str(e))
	+ try:
	+ search = api.api_search(None)
	+ search_res = search['search_res']
	+ search_stats = search['search_stats']
	+ except BadInputExc as e:
	+ message = str(e)

	env['search_stats'] = search_stats
	- env['responses'] = responses
	- env['messages'] = messages
	+ env['search_res'] = search_res
	+ env['message'] = message
	return render_template('upload_and_search.html', **env)

File Metadata

Mime Type: text/plain
Expires: Wed, Dec 18, 2:50 AM (1 d, 21 h ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3222118

D47.id157.diffNo OneTemporaryActions

D47.id157.diffView Options

File Metadata

Event Timeline

D47.id157.diff
No OneTemporary
Actions

D47.id157.diff
View Options