diff --git a/swh/web/ui/service.py b/swh/web/ui/service.py --- a/swh/web/ui/service.py +++ b/swh/web/ui/service.py @@ -6,7 +6,7 @@ from collections import defaultdict from swh.core import hashutil -from swh.web.ui import converters, query, upload, backend +from swh.web.ui import converters, query, backend from swh.web.ui.exc import NotFoundExc @@ -53,22 +53,6 @@ 'found': False} -def upload_and_search(file): - """Upload a file and compute its hash. - - """ - tmpdir, filename, filepath = upload.save_in_upload_folder(file) - res = {'filename': filename} - try: - content = hash_and_search(filepath) - res.update(content) - return res - finally: - # clean up - if tmpdir: - upload.cleanup(tmpdir) - - def lookup_hash(q): """Checks if the storage contains a given content checksum diff --git a/swh/web/ui/templates/upload_and_search.html b/swh/web/ui/templates/search.html rename from swh/web/ui/templates/upload_and_search.html rename to swh/web/ui/templates/search.html diff --git a/swh/web/ui/tests/test_service.py b/swh/web/ui/tests/test_service.py --- a/swh/web/ui/tests/test_service.py +++ b/swh/web/ui/tests/test_service.py @@ -317,33 +317,6 @@ self.storage.content_find.assert_called_once_with({'sha1': bhash}) mock_hashutil.hash_to_hex.assert_called_once_with(bhash) - @patch('swh.web.ui.service.upload') - @istest - def test_upload_and_search(self, mock_upload): - mock_upload.save_in_upload_folder.return_value = ( - '/tmp/dir', 'some-filename', '/tmp/dir/path/some-filename') - - service.hash_and_search = MagicMock(side_effect=lambda filepath: - {'sha1': 'blah', - 'found': True}) - mock_upload.cleanup.return_value = None - - file = MagicMock(filename='some-filename') - - # when - actual_res = service.upload_and_search(file) - - # then - self.assertEqual(actual_res, { - 'filename': 'some-filename', - 'sha1': 'blah', - 'found': True}) - - mock_upload.save_in_upload_folder.assert_called_with(file) - mock_upload.cleanup.assert_called_with('/tmp/dir') - service.hash_and_search.assert_called_once_with( - '/tmp/dir/path/some-filename') - @patch('swh.web.ui.service.backend') @istest def lookup_origin(self, mock_backend): diff --git a/swh/web/ui/tests/test_upload.py b/swh/web/ui/tests/test_upload.py deleted file mode 100644 --- a/swh/web/ui/tests/test_upload.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright (C) 2015 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU Affero General Public License version 3, or any later version -# See top-level LICENSE file for more information - -from nose.tools import istest -from unittest.mock import patch, MagicMock - -from swh.web.ui import upload -from swh.web.ui.tests import test_app - - -class UploadTestCase(test_app.SWHApiTestCase): - @istest - def allowed_file_ok(self): - # when - actual_perm = upload.allowed_file('README') - self.assertTrue(actual_perm) - - # when - actual_perm2 = upload.allowed_file('README', []) - self.assertTrue(actual_perm2) - - # when - actual_perm3 = upload.allowed_file('README', ['README', - 'LICENCE', - 'BUGS']) - self.assertTrue(actual_perm3) - - # when - actual_perm4 = upload.allowed_file('some-filename.txt', ['txt', - 'blah', - 'gz']) - self.assertTrue(actual_perm4) - - # when - actual_perm5 = upload.allowed_file('something.tar.gz', ['gz', - 'txt', - 'tar.gz']) - # then - self.assertTrue(actual_perm5) - - @istest - def allowed_file_denied(self): - # when - actual_perm = upload.allowed_file('some-filename', ['blah']) - self.assertFalse(actual_perm) - - # when - actual_perm = upload.allowed_file('something.tgz', ['gz', - 'txt', - 'tar.gz']) - # then - self.assertFalse(actual_perm) - - @patch('swh.web.ui.upload.os.path') - @patch('swh.web.ui.upload.shutil') - @istest - def cleanup_ok(self, mock_shutil, mock_os_path): - # given - mock_os_path.commonprefix.return_value = '/some/upload-dir' - mock_shutil.rmtree.return_value = True - - # when - upload.cleanup('/some/upload-dir/some-dummy-path') - - # then - mock_os_path.commonprefix.assert_called_with( - ['/some/upload-dir', '/some/upload-dir/some-dummy-path']) - mock_shutil.rmtree.assert_called_with( - '/some/upload-dir/some-dummy-path') - - @patch('swh.web.ui.upload.os.path') - @patch('swh.web.ui.upload.shutil') - @istest - def cleanup_should_fail(self, mock_shutil, mock_os_path): - # given - mock_os_path.commonprefix.return_value = '/somewhere/forbidden' - mock_shutil.rmtree.return_value = True - - # when - with self.assertRaises(AssertionError): - upload.cleanup('/some/upload-dir/some-dummy-path') - - # then - mock_os_path.commonprefix.assert_called_with( - ['/some/upload-dir', '/some/upload-dir/some-dummy-path']) - self.assertTrue(mock_shutil.rmtree.not_called) - - @istest - def save_in_upload_folder_no_file(self): - # when - act_tmpdir, act_name, act_path = upload.save_in_upload_folder(None) - - # then - self.assertIsNone(act_tmpdir) - self.assertIsNone(act_name) - self.assertIsNone(act_path) - - @istest - def save_in_upload_folder_file_not_allowed(self): - # given - file = MagicMock() - file.filename = 'some-non-file-allowed.ext' - - # when - with self.assertRaises(ValueError) as exc: - act_tmpdir, act_name, act_path = upload.save_in_upload_folder(file) - - # then - self.assertIn('Only', exc.exception.args[0]) - self.assertIn('extensions are valid for upload', exc.exception.args[0]) - - @patch('swh.web.ui.upload.werkzeug') - @patch('swh.web.ui.upload.tempfile') - @istest - def save_in_upload_folder_ok(self, mock_tempfile, mock_werkzeug): - # given - upload_folder = self.app_config['conf']['upload_folder'] - - # mock the dependencies - mock_werkzeug.secure_filename.return_value = 'some-allowed-file.txt' - tmpdir = upload_folder + '/foobar/' - mock_tempfile.mkdtemp.return_value = tmpdir - - # mock the input - file = MagicMock() - file.filename = 'some-allowed-file.txt' - - # when - act_tmpdir, act_name, act_path = upload.save_in_upload_folder(file) - - # then - expected_tmpdir = tmpdir - expected_filename = 'some-allowed-file.txt' - expected_filepath = tmpdir + 'some-allowed-file.txt' - - self.assertEqual(act_tmpdir, expected_tmpdir) - self.assertEqual(act_name, expected_filename) - self.assertEqual(act_path, expected_filepath) - - mock_werkzeug.secure_filename.assert_called_with(expected_filename) - file.save.assert_called_once_with(expected_filepath) - - mock_tempfile.mkdtemp.assert_called_with( - suffix='tmp', - prefix='swh.web.ui-', - dir=upload_folder) diff --git a/swh/web/ui/tests/views/test_api.py b/swh/web/ui/tests/views/test_api.py --- a/swh/web/ui/tests/views/test_api.py +++ b/swh/web/ui/tests/views/test_api.py @@ -526,51 +526,6 @@ mock_service.stat_origin_visits.assert_called_once_with(2) @patch('swh.web.ui.views.api.service') - @patch('swh.web.ui.views.api.request') - @istest - def api_uploadnsearch_bad_input(self, mock_request, mock_service): - # given - mock_request.files = {} - - # when - rv = self.app.post('/api/1/uploadnsearch/') - - self.assertEquals(rv.status_code, 400) - self.assertEquals(rv.mimetype, 'application/json') - - response_data = json.loads(rv.data.decode('utf-8')) - self.assertEquals(response_data, { - 'error': "Bad request, missing 'filename' entry in form."}) - - mock_service.upload_and_search.called = False - - @patch('swh.web.ui.views.api.service') - @patch('swh.web.ui.views.api.request') - @istest - def api_uploadnsearch(self, mock_request, mock_service): - # given - mock_request.files = {'filename': 'simple-filename'} - mock_service.upload_and_search.return_value = { - 'filename': 'simple-filename', - 'sha1': 'some-hex-sha1', - 'found': False, - } - - # when - rv = self.app.post('/api/1/uploadnsearch/') - - self.assertEquals(rv.status_code, 200) - self.assertEquals(rv.mimetype, 'application/json') - - response_data = json.loads(rv.data.decode('utf-8')) - self.assertEquals(response_data, {'filename': 'simple-filename', - 'sha1': 'some-hex-sha1', - 'found': False}) - - mock_service.upload_and_search.assert_called_once_with( - 'simple-filename') - - @patch('swh.web.ui.views.api.service') @istest def api_origin(self, mock_service): # given diff --git a/swh/web/ui/tests/views/test_browse.py b/swh/web/ui/tests/views/test_browse.py --- a/swh/web/ui/tests/views/test_browse.py +++ b/swh/web/ui/tests/views/test_browse.py @@ -28,7 +28,7 @@ self.assertEqual(rv.status_code, 200) self.assertEqual(self.get_context_variable('message'), '') self.assertEqual(self.get_context_variable('search_res'), None) - self.assert_template_used('upload_and_search.html') + self.assert_template_used('search.html') @patch('swh.web.ui.views.browse.api') @istest @@ -50,7 +50,7 @@ {'filename': None, 'sha1': 'sha1:456', 'found': False}]) - self.assert_template_used('upload_and_search.html') + self.assert_template_used('search.html') mock_api.api_search.assert_called_once_with('sha1:456') @@ -66,7 +66,7 @@ self.assertEqual(rv.status_code, 200) self.assertEqual(self.get_context_variable('message'), 'error msg') self.assertEqual(self.get_context_variable('search_res'), None) - self.assert_template_used('upload_and_search.html') + self.assert_template_used('search.html') mock_api.api_search.assert_called_once_with('sha1_git:789') @@ -91,7 +91,7 @@ self.assertTrue(resp is not None) self.assertEqual(resp['sha1'], 'sha1:123') self.assertEqual(resp['found'], True) - self.assert_template_used('upload_and_search.html') + self.assert_template_used('search.html') mock_api.api_search.assert_called_once_with('sha1:123') @@ -116,7 +116,7 @@ self.assertEqual(self.get_context_variable('search_res'), None) self.assertEqual(self.get_context_variable('message'), 'error bad input') - self.assert_template_used('upload_and_search.html') + self.assert_template_used('search.html') @patch('swh.web.ui.views.browse.request') @patch('swh.web.ui.views.browse.api') @@ -153,7 +153,7 @@ self.assertEqual(b['found'], False) self.assertEqual(self.get_context_variable('message'), '') - self.assert_template_used('upload_and_search.html') + self.assert_template_used('search.html') @patch('swh.web.ui.views.browse.request') @patch('swh.web.ui.views.browse.api') @@ -189,7 +189,7 @@ a, b = self.get_context_variable('search_res') self.assertEqual(a['found'], False) self.assertEqual(b['found'], True) - self.assert_template_used('upload_and_search.html') + self.assert_template_used('search.html') class ContentView(test_app.SWHViewTestCase): diff --git a/swh/web/ui/upload.py b/swh/web/ui/upload.py deleted file mode 100644 --- a/swh/web/ui/upload.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (C) 2015 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU Affero General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import os -import tempfile -import shutil -import werkzeug - -from swh.web.ui import main - - -def allowed_file(filename, allowed_extensions=[]): - """Filter on filename extension. - The filename to check for permission. - - Args: - filename. If no extension on the filename, the filename itself is - checked against allowed extensions (example of current extensionless - filenames: README, LICENCE, BUGS, etc...) - - Returns: - True if allowed, False otherwise. - - """ - if allowed_extensions == []: - return True - if '.' in filename: - return filename.rsplit('.', 1)[1] in allowed_extensions - return filename in allowed_extensions - - -def save_in_upload_folder(file): - """Persist uploaded file on server. - - Args: - File object (as per Flask's submission form) - - Returns: - a triplet: - - the temporary directory holding the persisted file - - the filename without any path from the file entry - - the complete path filepath - - """ - main_conf = main.app.config['conf'] - upload_folder = main_conf['upload_folder'] - allowed_extensions = main_conf['upload_allowed_extensions'] - - if not file: - return None, None, None - - filename = file.filename - if allowed_file(filename, allowed_extensions): - filename = werkzeug.secure_filename(filename) - - tmpdir = tempfile.mkdtemp(suffix='tmp', - prefix='swh.web.ui-', - dir=upload_folder) - - filepath = os.path.join(tmpdir, filename) - file.save(filepath) # persist on disk (not found how to avoid this) - - return tmpdir, filename, filepath - else: - raise ValueError( - 'Only %s extensions are valid for upload.' % allowed_extensions) - - -def cleanup(tmpdir): - """Clean up after oneself. - - Args: - The directory dir to destroy. - """ - upload_folder = main.app.config['conf']['upload_folder'] - assert (os.path.commonprefix([upload_folder, tmpdir]) == upload_folder) - shutil.rmtree(tmpdir) diff --git a/swh/web/ui/views/api.py b/swh/web/ui/views/api.py --- a/swh/web/ui/views/api.py +++ b/swh/web/ui/views/api.py @@ -8,7 +8,7 @@ from flask import request, url_for, Response, redirect from swh.web.ui import service, utils -from swh.web.ui.exc import BadInputExc, NotFoundExc +from swh.web.ui.exc import NotFoundExc from swh.web.ui.main import app @@ -946,26 +946,3 @@ lookup_fn=service.lookup_entity_by_uuid, error_msg_if_not_found="Entity with uuid '%s' not found." % uuid, enrich_fn=utils.enrich_entity) - - -@app.route('/api/1/uploadnsearch/', methods=['POST']) -def api_uploadnsearch(): - """Upload the file's content in the post body request. - Compute its hash and determine if it exists in the storage. - - Args: - request.files filled with the filename's data to upload. - - Returns: - Dictionary with 'sha1', 'filename' and 'found' predicate depending - on whether we find it or not. - - Raises: - BadInputExc in case of the form submitted is incorrect. - - """ - file = request.files.get('filename') - if not file: - raise BadInputExc("Bad request, missing 'filename' entry in form.") - - return service.upload_and_search(file) diff --git a/swh/web/ui/views/browse.py b/swh/web/ui/views/browse.py --- a/swh/web/ui/views/browse.py +++ b/swh/web/ui/views/browse.py @@ -71,7 +71,7 @@ env['search_stats'] = search_stats env['search_res'] = search_res env['message'] = message - return render_template('upload_and_search.html', **env) + return render_template('search.html', **env) @app.route('/browse/content/')