diff --git a/swh/web/api/apiresponse.py b/swh/web/api/apiresponse.py index 734c25d9..1d47b513 100644 --- a/swh/web/api/apiresponse.py +++ b/swh/web/api/apiresponse.py @@ -1,190 +1,192 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import traceback from django.utils.html import escape from rest_framework.response import Response from swh.storage.exc import StorageDBError, StorageAPIError from swh.web.api import utils -from swh.web.common.exc import NotFoundExc, ForbiddenExc +from swh.web.common.exc import NotFoundExc, ForbiddenExc, BadInputExc from swh.web.common.utils import shorten_path, gen_path_info from swh.web.config import get_config def compute_link_header(rv, options): """Add Link header in returned value results. Args: rv (dict): dictionary with keys: - headers: potential headers with 'link-next' and 'link-prev' keys - results: containing the result to return options (dict): the initial dict to update with result if any Returns: dict: dictionary with optional keys 'link-next' and 'link-prev' """ link_headers = [] if 'headers' not in rv: return {} rv_headers = rv['headers'] if 'link-next' in rv_headers: link_headers.append('<%s>; rel="next"' % ( rv_headers['link-next'])) if 'link-prev' in rv_headers: link_headers.append('<%s>; rel="previous"' % ( rv_headers['link-prev'])) if link_headers: link_header_str = ','.join(link_headers) headers = options.get('headers', {}) headers.update({ 'Link': link_header_str }) return headers return {} def filter_by_fields(request, data): """Extract a request parameter 'fields' if it exists to permit the filtering on the data dict's keys. If such field is not provided, returns the data as is. """ fields = request.query_params.get('fields') if fields: fields = set(fields.split(',')) data = utils.filter_field_keys(data, fields) return data def transform(rv): """Transform an eventual returned value with multiple layer of information with only what's necessary. If the returned value rv contains the 'results' key, this is the associated value which is returned. Otherwise, return the initial dict without the potential 'headers' key. """ if 'results' in rv: return rv['results'] if 'headers' in rv: rv.pop('headers') return rv def make_api_response(request, data, doc_data={}, options={}): """Generates an API response based on the requested mimetype. Args: request: a DRF Request object data: raw data to return in the API response doc_data: documentation data for HTML response options: optional data that can be used to generate the response Returns: a DRF Response a object """ if data: options['headers'] = compute_link_header(data, options) data = transform(data) data = filter_by_fields(request, data) doc_env = doc_data headers = {} if 'headers' in options: doc_env['headers_data'] = options['headers'] headers = options['headers'] # get request status code doc_env['status_code'] = options.get('status', 200) response_args = {'status': doc_env['status_code'], 'headers': headers, 'content_type': request.accepted_media_type} # when requesting HTML, typically when browsing the API through its # documented views, we need to enrich the input data with documentation # related ones and inform DRF that we request HTML template rendering if request.accepted_media_type == 'text/html': if data: data = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')) doc_env['response_data'] = data doc_env['request'] = { 'path': request.path, 'method': request.method, 'absolute_uri': request.build_absolute_uri(), } doc_env['heading'] = shorten_path(str(request.path)) if 'route' in doc_env: doc_env['endpoint_path'] = gen_path_info(doc_env['route']) response_args['data'] = doc_env response_args['template_name'] = 'api/apidoc.html' # otherwise simply return the raw data and let DRF picks # the correct renderer (JSON or YAML) else: response_args['data'] = data return Response(**response_args) def error_response(request, error, doc_data): """Private function to create a custom error response. Args: request: a DRF Request object error: the exception that caused the error doc_data: documentation data for HTML response """ - error_code = 400 - if isinstance(error, NotFoundExc): + error_code = 500 + if isinstance(error, BadInputExc): + error_code = 400 + elif isinstance(error, NotFoundExc): error_code = 404 elif isinstance(error, ForbiddenExc): error_code = 403 elif isinstance(error, StorageDBError): error_code = 503 elif isinstance(error, StorageAPIError): error_code = 503 error_opts = {'status': error_code} error_data = { 'exception': error.__class__.__name__, 'reason': str(error), } if request.accepted_media_type == 'text/html': error_data['reason'] = escape(error_data['reason']) if get_config()['debug']: error_data['traceback'] = traceback.format_exc() return make_api_response(request, error_data, doc_data, options=error_opts) diff --git a/swh/web/tests/api/test_apidoc.py b/swh/web/tests/api/test_apidoc.py index 9d1fcc52..358bd0a1 100644 --- a/swh/web/tests/api/test_apidoc.py +++ b/swh/web/tests/api/test_apidoc.py @@ -1,265 +1,297 @@ # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from rest_framework.test import APITestCase from rest_framework.response import Response +from swh.storage.exc import StorageDBError, StorageAPIError + from swh.web.api.apidoc import api_doc, _parse_httpdomain_doc from swh.web.api.apiurls import api_route +from swh.web.common.exc import BadInputExc, ForbiddenExc, NotFoundExc from swh.web.tests.testcase import WebTestCase # flake8: noqa httpdomain_doc = """ .. http:get:: /api/1/revision/(sha1_git)/ Get information about a revision in the archive. Revisions are identified by **sha1** checksums, compatible with Git commit identifiers. See :func:`swh.model.identifiers.revision_identifier` in our data model module for details about how they are computed. :param string sha1_git: hexadecimal representation of the revision **sha1_git** identifier :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :>json object author: information about the author of the revision :>json object committer: information about the committer of the revision :>json string committer_date: ISO representation of the commit date (in UTC) :>json string date: ISO representation of the revision date (in UTC) :>json string directory: the unique identifier that revision points to :>json string directory_url: link to :http:get:`/api/1/directory/(sha1_git)/[(path)/]` to get information about the directory associated to the revision :>json string id: the revision unique identifier :>json boolean merge: whether or not the revision corresponds to a merge commit :>json string message: the message associated to the revision :>json array parents: the parents of the revision, i.e. the previous revisions that head directly to it, each entry of that array contains an unique parent revision identifier but also a link to :http:get:`/api/1/revision/(sha1_git)/` to get more information about it :>json string type: the type of the revision **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid **sha1_git** value has been provided :statuscode 404: requested revision can not be found in the archive **Request:** .. parsed-literal:: $ curl -i :swh_web_api:`revision/aafb16d69fd30ff58afdd69036a26047f3aebdc6/` """ class APIDocTestCase(WebTestCase, APITestCase): + exception_http_code = { + BadInputExc: 400, + ForbiddenExc: 403, + NotFoundExc: 404, + Exception: 500, + StorageAPIError: 503, + StorageDBError: 503, + } + def test_apidoc_nodoc_failure(self): with self.assertRaises(Exception): @api_doc('/my/nodoc/url/') def apidoc_nodoc_tester(request, arga=0, argb=0): return Response(arga + argb) @staticmethod @api_route(r'/some/(?P[0-9]+)/(?P[0-9]+)/', 'some-doc-route') @api_doc('/some/doc/route/') def apidoc_route(request, myarg, myotherarg, akw=0): """ Sample doc """ return {'result': int(myarg) + int(myotherarg) + akw} def test_apidoc_route_doc(self): # when rv = self.client.get('/api/1/some/doc/route/') # then self.assertEqual(rv.status_code, 200, rv.data) self.assertTemplateUsed('api/apidoc.html') def test_apidoc_route_fn(self): # when rv = self.client.get('/api/1/some/1/1/') # then self.assertEqual(rv.status_code, 200, rv.data) + @staticmethod + @api_route(r'/test/error/(?P.+)/', + 'test-error') + @api_doc('/test/error/') + def apidoc_test_error_route(request, exc_name): + """ + Sample doc + """ + for e in APIDocTestCase.exception_http_code.keys(): + if e.__name__ == exc_name: + raise e('Error') + + def test_apidoc_error(self): + for exc, code in self.exception_http_code.items(): + # when + rv = self.client.get('/api/1/test/error/%s/' % exc.__name__) + + # then + self.assertEqual(rv.status_code, code) + @staticmethod @api_route(r'/some/full/(?P[0-9]+)/(?P[0-9]+)/', 'some-complete-doc-route') @api_doc('/some/complete/doc/route/') def apidoc_full_stack(request, myarg, myotherarg, akw=0): """ Sample doc """ return {'result': int(myarg) + int(myotherarg) + akw} def test_apidoc_full_stack_doc(self): # when rv = self.client.get('/api/1/some/complete/doc/route/') # then self.assertEqual(rv.status_code, 200, rv.data) self.assertTemplateUsed('api/apidoc.html') def test_apidoc_full_stack_fn(self): # when rv = self.client.get('/api/1/some/full/1/1/') # then self.assertEqual(rv.status_code, 200, rv.data) def test_api_doc_parse_httpdomain(self): doc_data = { 'description': '', 'urls': [], 'args': [], 'params': [], 'resheaders': [], 'reqheaders': [], 'return_type': '', 'returns': [], 'status_codes': [], 'examples': [] } _parse_httpdomain_doc(httpdomain_doc, doc_data) expected_urls = [{ 'rule': '/api/1/revision/ **\\(sha1_git\\)** /', 'methods': ['GET', 'HEAD', 'OPTIONS'] }] self.assertIn('urls', doc_data) self.assertEqual(doc_data['urls'], expected_urls) expected_description = 'Get information about a revision in the archive. \ Revisions are identified by **sha1** checksums, compatible with Git commit \ identifiers. See **swh.model.identifiers.revision_identifier** in our data \ model module for details about how they are computed.' self.assertIn('description', doc_data) self.assertEqual(doc_data['description'], expected_description) expected_args = [{ 'name': 'sha1_git', 'type': 'string', 'doc': 'hexadecimal representation of the revision **sha1_git** identifier' }] self.assertIn('args', doc_data) self.assertEqual(doc_data['args'], expected_args) expected_params = [] self.assertIn('params', doc_data) self.assertEqual(doc_data['params'], expected_params) expected_reqheaders = [{ 'doc': 'the requested response content type, either ``application/json`` or ``application/yaml``', 'name': 'Accept' }] self.assertIn('reqheaders', doc_data) self.assertEqual(doc_data['reqheaders'], expected_reqheaders) expected_resheaders = [{ 'doc': 'this depends on **Accept** header of request', 'name': 'Content-Type' }] self.assertIn('resheaders', doc_data) self.assertEqual(doc_data['resheaders'], expected_resheaders) expected_statuscodes = [ { 'code': '200', 'doc': 'no error' }, { 'code': '400', 'doc': 'an invalid **sha1_git** value has been provided' }, { 'code': '404', 'doc': 'requested revision can not be found in the archive' } ] self.assertIn('status_codes', doc_data) self.assertEqual(doc_data['status_codes'], expected_statuscodes) expected_return_type = 'object' self.assertIn('return_type', doc_data) self.assertEqual(doc_data['return_type'], expected_return_type) expected_returns = [ { 'name': 'author', 'type': 'object', 'doc': 'information about the author of the revision' }, { 'name': 'committer', 'type': 'object', 'doc': 'information about the committer of the revision' }, { 'name': 'committer_date', 'type': 'string', 'doc': 'ISO representation of the commit date (in UTC)' }, { 'name': 'date', 'type': 'string', 'doc': 'ISO representation of the revision date (in UTC)' }, { 'name': 'directory', 'type': 'string', 'doc': 'the unique identifier that revision points to' }, { 'name': 'directory_url', 'type': 'string', 'doc': 'link to ``_ to get information about the directory associated to the revision' }, { 'name': 'id', 'type': 'string', 'doc': 'the revision unique identifier' }, { 'name': 'merge', 'type': 'boolean', 'doc': 'whether or not the revision corresponds to a merge commit' }, { 'name': 'message', 'type': 'string', 'doc': 'the message associated to the revision' }, { 'name': 'parents', 'type': 'array', 'doc': 'the parents of the revision, i.e. the previous revisions that head directly to it, each entry of that array contains an unique parent revision identifier but also a link to ``_ to get more information about it' }, { 'name': 'type', 'type': 'string', 'doc': 'the type of the revision' } ] self.assertIn('returns', doc_data) self.assertEqual(doc_data['returns'], expected_returns) expected_examples = ['/api/1/revision/aafb16d69fd30ff58afdd69036a26047f3aebdc6/'] self.assertIn('examples', doc_data) self.assertEqual(doc_data['examples'], expected_examples) diff --git a/swh/web/tests/api/views/test_origin.py b/swh/web/tests/api/views/test_origin.py index c528e1f8..74fe25ce 100644 --- a/swh/web/tests/api/views/test_origin.py +++ b/swh/web/tests/api/views/test_origin.py @@ -1,635 +1,636 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from unittest.mock import patch from hypothesis import given, strategies import pytest from requests.utils import parse_header_links from rest_framework.test import APITestCase from swh.storage.exc import StorageDBError, StorageAPIError +from swh.web.common.exc import BadInputExc from swh.web.common.utils import reverse from swh.web.common.origin_visits import get_origin_visits from swh.web.tests.strategies import ( origin, new_origin, visit_dates, new_snapshots ) from swh.web.tests.testcase import WebTestCase from swh.web.tests.data import get_tests_data class OriginApiTestCase(WebTestCase, APITestCase): def _scroll_results(self, url): """Iterates through pages of results, and returns them all.""" results = [] while True: rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') results.extend(rv.data) if 'Link' in rv: for link in parse_header_links(rv['Link']): if link['rel'] == 'next': # Found link to next page of results url = link['url'] break else: # No link with 'rel=next' break else: # No Link header break return results @patch('swh.web.api.views.origin.get_origin_visits') def test_api_lookup_origin_visits_raise_error( self, mock_get_origin_visits, ): err_msg = 'voluntary error to check the bad request middleware.' - mock_get_origin_visits.side_effect = ValueError(err_msg) + mock_get_origin_visits.side_effect = BadInputExc(err_msg) url = reverse( 'api-1-origin-visits', url_args={'origin_url': 'http://foo'}) rv = self.client.get(url) self.assertEqual(rv.status_code, 400, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { - 'exception': 'ValueError', + 'exception': 'BadInputExc', 'reason': err_msg}) @patch('swh.web.api.views.origin.get_origin_visits') def test_api_lookup_origin_visits_raise_swh_storage_error_db( self, mock_get_origin_visits): err_msg = 'Storage exploded! Will be back online shortly!' mock_get_origin_visits.side_effect = StorageDBError(err_msg) url = reverse( 'api-1-origin-visits', url_args={'origin_url': 'http://foo'}) rv = self.client.get(url) self.assertEqual(rv.status_code, 503, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { 'exception': 'StorageDBError', 'reason': 'An unexpected error occurred in the backend: %s' % err_msg}) @patch('swh.web.api.views.origin.get_origin_visits') def test_api_lookup_origin_visits_raise_swh_storage_error_api( self, mock_get_origin_visits): err_msg = 'Storage API dropped dead! Will resurrect asap!' mock_get_origin_visits.side_effect = StorageAPIError(err_msg) url = reverse( 'api-1-origin-visits', url_args={'origin_url': 'http://foo'}) rv = self.client.get(url) self.assertEqual(rv.status_code, 503, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { 'exception': 'StorageAPIError', 'reason': 'An unexpected error occurred in the api backend: %s' % err_msg }) @given(new_origin(), visit_dates(3), new_snapshots(3)) def test_api_lookup_origin_visits(self, new_origin, visit_dates, new_snapshots): self.storage.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): origin_visit = self.storage.origin_visit_add( new_origin['url'], visit_date, type='git') self.storage.snapshot_add([new_snapshots[i]]) self.storage.origin_visit_update( new_origin['url'], origin_visit['visit'], snapshot=new_snapshots[i]['id']) all_visits = list(reversed(get_origin_visits(new_origin))) for last_visit, expected_visits in ( (None, all_visits[:2]), (all_visits[1]['visit'], all_visits[2:4])): url = reverse('api-1-origin-visits', url_args={'origin_url': new_origin['url']}, query_params={'per_page': 2, 'last_visit': last_visit}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') for expected_visit in expected_visits: origin_visit_url = reverse( 'api-1-origin-visit', url_args={'origin_url': new_origin['url'], 'visit_id': expected_visit['visit']}) snapshot_url = reverse( 'api-1-snapshot', url_args={'snapshot_id': expected_visit['snapshot']}) expected_visit['origin'] = new_origin['url'] expected_visit['origin_visit_url'] = origin_visit_url expected_visit['snapshot_url'] = snapshot_url self.assertEqual(rv.data, expected_visits) @given(new_origin(), visit_dates(3), new_snapshots(3)) def test_api_lookup_origin_visits_by_id(self, new_origin, visit_dates, new_snapshots): self.storage.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): origin_visit = self.storage.origin_visit_add( new_origin['url'], visit_date, type='git') self.storage.snapshot_add([new_snapshots[i]]) self.storage.origin_visit_update( new_origin['url'], origin_visit['visit'], snapshot=new_snapshots[i]['id']) all_visits = list(reversed(get_origin_visits(new_origin))) for last_visit, expected_visits in ( (None, all_visits[:2]), (all_visits[1]['visit'], all_visits[2:4])): url = reverse('api-1-origin-visits', url_args={'origin_url': new_origin['url']}, query_params={'per_page': 2, 'last_visit': last_visit}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') for expected_visit in expected_visits: origin_visit_url = reverse( 'api-1-origin-visit', url_args={'origin_url': new_origin['url'], 'visit_id': expected_visit['visit']}) snapshot_url = reverse( 'api-1-snapshot', url_args={'snapshot_id': expected_visit['snapshot']}) expected_visit['origin'] = new_origin['url'] expected_visit['origin_visit_url'] = origin_visit_url expected_visit['snapshot_url'] = snapshot_url self.assertEqual(rv.data, expected_visits) @given(new_origin(), visit_dates(3), new_snapshots(3)) def test_api_lookup_origin_visit(self, new_origin, visit_dates, new_snapshots): self.storage.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): origin_visit = self.storage.origin_visit_add( new_origin['url'], visit_date, type='git') visit_id = origin_visit['visit'] self.storage.snapshot_add([new_snapshots[i]]) self.storage.origin_visit_update( new_origin['url'], origin_visit['visit'], snapshot=new_snapshots[i]['id']) url = reverse('api-1-origin-visit', url_args={'origin_url': new_origin['url'], 'visit_id': visit_id}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') expected_visit = self.origin_visit_get_by( new_origin['url'], visit_id) origin_url = reverse('api-1-origin', url_args={'origin_url': new_origin['url']}) snapshot_url = reverse( 'api-1-snapshot', url_args={'snapshot_id': expected_visit['snapshot']}) expected_visit['origin'] = new_origin['url'] expected_visit['origin_url'] = origin_url expected_visit['snapshot_url'] = snapshot_url self.assertEqual(rv.data, expected_visit) @given(new_origin(), visit_dates(2), new_snapshots(1)) def test_api_lookup_origin_visit_latest( self, new_origin, visit_dates, new_snapshots): self.storage.origin_add_one(new_origin) visit_dates.sort() visit_ids = [] for i, visit_date in enumerate(visit_dates): origin_visit = self.storage.origin_visit_add( new_origin['url'], visit_date, type='git') visit_ids.append(origin_visit['visit']) self.storage.snapshot_add([new_snapshots[0]]) self.storage.origin_visit_update( new_origin['url'], visit_ids[0], snapshot=new_snapshots[0]['id']) url = reverse('api-1-origin-visit-latest', url_args={'origin_url': new_origin['url']}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') expected_visit = self.origin_visit_get_by( new_origin['url'], visit_ids[1]) origin_url = reverse('api-1-origin', url_args={'origin_url': new_origin['url']}) expected_visit['origin'] = new_origin['url'] expected_visit['origin_url'] = origin_url expected_visit['snapshot_url'] = None self.assertEqual(rv.data, expected_visit) @given(new_origin(), visit_dates(2), new_snapshots(1)) def test_api_lookup_origin_visit_latest_with_snapshot( self, new_origin, visit_dates, new_snapshots): self.storage.origin_add_one(new_origin) visit_dates.sort() visit_ids = [] for i, visit_date in enumerate(visit_dates): origin_visit = self.storage.origin_visit_add( new_origin['url'], visit_date, type='git') visit_ids.append(origin_visit['visit']) self.storage.snapshot_add([new_snapshots[0]]) self.storage.origin_visit_update( new_origin['url'], visit_ids[0], snapshot=new_snapshots[0]['id']) url = reverse('api-1-origin-visit-latest', url_args={'origin_url': new_origin['url']}) url += '?require_snapshot=true' rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') expected_visit = self.origin_visit_get_by( new_origin['url'], visit_ids[0]) origin_url = reverse('api-1-origin', url_args={'origin_url': new_origin['url']}) snapshot_url = reverse( 'api-1-snapshot', url_args={'snapshot_id': expected_visit['snapshot']}) expected_visit['origin'] = new_origin['url'] expected_visit['origin_url'] = origin_url expected_visit['snapshot_url'] = snapshot_url self.assertEqual(rv.data, expected_visit) @given(origin()) def test_api_lookup_origin_visit_not_found(self, origin): all_visits = list(reversed(get_origin_visits(origin))) max_visit_id = max([v['visit'] for v in all_visits]) url = reverse('api-1-origin-visit', url_args={'origin_url': origin['url'], 'visit_id': max_visit_id + 1}) rv = self.client.get(url) self.assertEqual(rv.status_code, 404, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { 'exception': 'NotFoundExc', 'reason': 'Origin %s or its visit with id %s not found!' % (origin['url'], max_visit_id+1) }) @pytest.mark.origin_id def test_api_origins(self): origins = get_tests_data()['origins'] origin_urls = {origin['url'] for origin in origins} # Get only one url = reverse('api-1-origins', query_params={'origin_count': 1}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), 1) self.assertLess({origin['url'] for origin in rv.data}, origin_urls) # Get all url = reverse('api-1-origins', query_params={'origin_count': len(origins)}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), len(origins)) self.assertEqual({origin['url'] for origin in rv.data}, origin_urls) # Get "all + 10" url = reverse('api-1-origins', query_params={'origin_count': len(origins)+10}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), len(origins)) self.assertEqual({origin['url'] for origin in rv.data}, origin_urls) @pytest.mark.origin_id @given(strategies.integers(min_value=1)) def test_api_origins_scroll(self, origin_count): origins = get_tests_data()['origins'] origin_urls = {origin['url'] for origin in origins} url = reverse('api-1-origins', query_params={'origin_count': origin_count}) results = self._scroll_results(url) self.assertEqual(len(results), len(origins)) self.assertEqual({origin['url'] for origin in results}, origin_urls) @given(origin()) def test_api_origin_by_url(self, origin): url = reverse('api-1-origin', url_args={'origin_url': origin['url']}) rv = self.client.get(url) expected_origin = self.origin_get(origin) origin_visits_url = reverse('api-1-origin-visits', url_args={'origin_url': origin['url']}) expected_origin['origin_visits_url'] = origin_visits_url self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, expected_origin) @given(new_origin()) def test_api_origin_not_found(self, new_origin): url = reverse('api-1-origin', url_args={'origin_url': new_origin['url']}) rv = self.client.get(url) self.assertEqual(rv.status_code, 404, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { 'exception': 'NotFoundExc', 'reason': 'Origin with url %s not found!' % new_origin['url'] }) @pytest.mark.origin_id def test_api_origin_search(self): expected_origins = { 'https://github.com/wcoder/highlightjs-line-numbers.js', 'https://github.com/memononen/libtess2', } # Search for 'github.com', get only one url = reverse('api-1-origin-search', url_args={'url_pattern': 'github.com'}, query_params={'limit': 1}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), 1) self.assertLess({origin['url'] for origin in rv.data}, expected_origins) # Search for 'github.com', get all url = reverse('api-1-origin-search', url_args={'url_pattern': 'github.com'}, query_params={'limit': 2}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual({origin['url'] for origin in rv.data}, expected_origins) # Search for 'github.com', get more than available url = reverse('api-1-origin-search', url_args={'url_pattern': 'github.com'}, query_params={'limit': 10}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual({origin['url'] for origin in rv.data}, expected_origins) @pytest.mark.origin_id def test_api_origin_search_regexp(self): expected_origins = { 'https://github.com/memononen/libtess2', 'repo_with_submodules' } url = reverse('api-1-origin-search', url_args={'url_pattern': '(repo|libtess)'}, query_params={'limit': 10, 'regexp': True}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual({origin['url'] for origin in rv.data}, expected_origins) @pytest.mark.origin_id @given(strategies.integers(min_value=1)) def test_api_origin_search_scroll(self, limit): expected_origins = { 'https://github.com/wcoder/highlightjs-line-numbers.js', 'https://github.com/memononen/libtess2', } url = reverse('api-1-origin-search', url_args={'url_pattern': 'github.com'}, query_params={'limit': limit}) results = self._scroll_results(url) self.assertEqual({origin['url'] for origin in results}, expected_origins) def test_api_origin_search_limit(self): self.storage.origin_add([ {'url': 'http://foobar/{}'.format(i)} for i in range(2000) ]) url = reverse('api-1-origin-search', url_args={'url_pattern': 'foobar'}, query_params={'limit': 1050}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), 1000) @given(origin()) def test_api_origin_metadata_search(self, origin): with patch('swh.web.common.service.idx_storage') as mock_idx_storage: mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .side_effect = lambda conjunction, limit: [{ 'from_revision': ( b'p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed ' b'\xf2U\xfa\x05B8'), 'metadata': {'author': 'Jane Doe'}, 'origin_url': origin['url'], 'tool': { 'configuration': { 'context': ['NpmMapping', 'CodemetaMapping'], 'type': 'local' }, 'id': 3, 'name': 'swh-metadata-detector', 'version': '0.0.1' } }] url = reverse('api-1-origin-metadata-search', query_params={'fulltext': 'Jane Doe'}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.content) self.assertEqual(rv['Content-Type'], 'application/json') expected_data = [{ 'url': origin['url'], 'metadata': { 'metadata': {'author': 'Jane Doe'}, 'from_revision': ( '7026b7c1a2af56521e951c01ed20f255fa054238'), 'tool': { 'configuration': { 'context': ['NpmMapping', 'CodemetaMapping'], 'type': 'local' }, 'id': 3, 'name': 'swh-metadata-detector', 'version': '0.0.1', } } }] actual_data = rv.data for d in actual_data: if 'id' in d: del d['id'] self.assertEqual(rv.data, expected_data) mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .assert_called_with(conjunction=['Jane Doe'], limit=70) @given(origin()) def test_api_origin_metadata_search_limit(self, origin): with patch('swh.web.common.service.idx_storage') as mock_idx_storage: mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .side_effect = lambda conjunction, limit: [{ 'from_revision': ( b'p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed ' b'\xf2U\xfa\x05B8'), 'metadata': {'author': 'Jane Doe'}, 'origin_url': origin['url'], 'tool': { 'configuration': { 'context': ['NpmMapping', 'CodemetaMapping'], 'type': 'local' }, 'id': 3, 'name': 'swh-metadata-detector', 'version': '0.0.1' } }] url = reverse('api-1-origin-metadata-search', query_params={'fulltext': 'Jane Doe'}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.content) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), 1) mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .assert_called_with(conjunction=['Jane Doe'], limit=70) url = reverse('api-1-origin-metadata-search', query_params={'fulltext': 'Jane Doe', 'limit': 10}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.content) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), 1) mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .assert_called_with(conjunction=['Jane Doe'], limit=10) url = reverse('api-1-origin-metadata-search', query_params={'fulltext': 'Jane Doe', 'limit': 987}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.content) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), 1) mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .assert_called_with(conjunction=['Jane Doe'], limit=100) @given(origin()) def test_api_origin_intrinsic_metadata(self, origin): with patch('swh.web.common.service.idx_storage') as mock_idx_storage: mock_idx_storage.origin_intrinsic_metadata_get \ .side_effect = lambda origin_urls: [{ 'from_revision': ( b'p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed ' b'\xf2U\xfa\x05B8'), 'metadata': {'author': 'Jane Doe'}, 'origin_url': origin['url'], 'tool': { 'configuration': { 'context': ['NpmMapping', 'CodemetaMapping'], 'type': 'local' }, 'id': 3, 'name': 'swh-metadata-detector', 'version': '0.0.1' } }] url = reverse('api-origin-intrinsic-metadata', url_args={'origin_url': origin['url']}) rv = self.client.get(url) mock_idx_storage.origin_intrinsic_metadata_get \ .assert_called_once_with([origin['url']]) self.assertEqual(rv.status_code, 200, rv.content) self.assertEqual(rv['Content-Type'], 'application/json') expected_data = {'author': 'Jane Doe'} self.assertEqual(rv.data, expected_data) @patch('swh.web.common.service.idx_storage') def test_api_origin_metadata_search_invalid(self, mock_idx_storage): url = reverse('api-1-origin-metadata-search') rv = self.client.get(url) self.assertEqual(rv.status_code, 400, rv.content) mock_idx_storage.assert_not_called() diff --git a/swh/web/tests/api/views/test_stat.py b/swh/web/tests/api/views/test_stat.py index 1e8ba052..60f35986 100644 --- a/swh/web/tests/api/views/test_stat.py +++ b/swh/web/tests/api/views/test_stat.py @@ -1,75 +1,76 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from rest_framework.test import APITestCase from unittest.mock import patch from swh.storage.exc import StorageDBError, StorageAPIError +from swh.web.common.exc import BadInputExc from swh.web.common.utils import reverse from swh.web.tests.testcase import WebTestCase class StatApiTestCase(WebTestCase, APITestCase): @patch('swh.web.api.views.stat.service') def test_api_1_stat_counters_raise_error(self, mock_service): - mock_service.stat_counters.side_effect = ValueError( + mock_service.stat_counters.side_effect = BadInputExc( 'voluntary error to check the bad request middleware.') url = reverse('api-1-stat-counters') rv = self.client.get(url) self.assertEqual(rv.status_code, 400, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { - 'exception': 'ValueError', + 'exception': 'BadInputExc', 'reason': 'voluntary error to check the bad request middleware.'}) @patch('swh.web.api.views.stat.service') def test_api_1_stat_counters_raise_from_db(self, mock_service): mock_service.stat_counters.side_effect = StorageDBError( 'Storage exploded! Will be back online shortly!') url = reverse('api-1-stat-counters') rv = self.client.get(url) self.assertEqual(rv.status_code, 503, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { 'exception': 'StorageDBError', 'reason': 'An unexpected error occurred in the backend: ' 'Storage exploded! Will be back online shortly!'}) @patch('swh.web.api.views.stat.service') def test_api_1_stat_counters_raise_from_api(self, mock_service): mock_service.stat_counters.side_effect = StorageAPIError( 'Storage API dropped dead! Will resurrect from its ashes asap!' ) url = reverse('api-1-stat-counters') rv = self.client.get(url) self.assertEqual(rv.status_code, 503, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { 'exception': 'StorageAPIError', 'reason': 'An unexpected error occurred in the api backend: ' 'Storage API dropped dead! Will resurrect from its ashes asap!' }) def test_api_1_stat_counters(self): url = reverse('api-1-stat-counters') rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, self.storage.stat_counters())