diff --git a/pytest.ini b/pytest.ini --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,3 @@ [pytest] norecursedirs = docs node_modules .tox DJANGO_SETTINGS_MODULE = swh.web.settings.tests -markers = - origin_id: execute tests using an origin id (deselect with '-m "not origin_id"') diff --git a/swh/web/api/views/origin.py b/swh/web/api/views/origin.py --- a/swh/web/api/views/origin.py +++ b/swh/web/api/views/origin.py @@ -134,10 +134,9 @@ @api_route(r'/origin/(?P[a-z]+)/url/(?P.+)/', 'api-1-origin') @api_route(r'/origin/(?P.+)/get/', 'api-1-origin') -@api_route(r'/origin/(?P[0-9]+)/', 'api-1-origin') @api_doc('/origin/') @format_docstring(return_origin=DOC_RETURN_ORIGIN) -def api_origin(request, origin_id=None, origin_type=None, origin_url=None): +def api_origin(request, origin_url, origin_type=None): """ .. http:get:: /api/1/origin/(origin_url)/get/ @@ -161,34 +160,6 @@ :swh_web_api:`origin/git/url/https://github.com/python/cpython/` - .. http:get:: /api/1/origin/(origin_id)/ - - Get information about a software origin. - - .. warning:: - - All endpoints using an ``origin_id`` or an ``origin_type`` are - deprecated and will be removed in the near future. Only those - using an ``origin_url`` will remain available. - You should use :http:get:`/api/1/origin/(origin_url)/get/` instead. - - :param int origin_id: a software origin identifier - - {return_origin} - - {common_headers} - - **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, - :http:method:`options` - - :statuscode 200: no error - :statuscode 404: requested origin can not be found in the archive - - **Example:** - - .. parsed-literal:: - - :swh_web_api:`origin/1/` .. http:get:: /api/1/origin/(origin_type)/url/(origin_url)/ @@ -196,9 +167,9 @@ .. warning:: - All endpoints using an ``origin_id`` or an ``origin_type`` are + All endpoints using an ``origin_type`` are deprecated and will be removed in the near future. Only those - using an ``origin_url`` will remain available. + using only an ``origin_url`` will remain available. You should use :http:get:`/api/1/origin/(origin_url)/get/` instead. :param string origin_type: the origin type (possible values are @@ -223,7 +194,6 @@ :swh_web_api:`origin/git/url/https://github.com/python/cpython/` """ ori_dict = { - 'id': int(origin_id) if origin_id else None, 'type': origin_type, 'url': origin_url } @@ -351,11 +321,10 @@ @api_route(r'/origin/(?P.*)/visits/', 'api-1-origin-visits') -@api_route(r'/origin/(?P[0-9]+)/visits/', 'api-1-origin-visits') @api_doc('/origin/visits/') @format_docstring( return_origin_visit_array=DOC_RETURN_ORIGIN_VISIT_ARRAY) -def api_origin_visits(request, origin_id=None, origin_url=None): +def api_origin_visits(request, origin_url): """ .. http:get:: /api/1/origin/(origin_url)/visits/ @@ -386,51 +355,11 @@ :swh_web_api:`origin/https://github.com/hylang/hy/visits/` - .. http:get:: /api/1/origin/(origin_id)/visits/ - - Get information about all visits of a software origin. - Visits are returned sorted in descending order according - to their date. - - .. warning:: - - All endpoints using an ``origin_id`` are deprecated and will be - removed in the near future. Only those using an ``origin_url`` - will remain available. - Use :http:get:`/api/1/origin/(origin_url)/visits/` instead. - - :param int origin_id: a software origin identifier - :query int per_page: specify the number of visits to list, for - pagination purposes - :query int last_visit: visit to start listing from, for pagination - purposes - - {common_headers} - {resheader_link} - - {return_origin_visit_array} - - **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, - :http:method:`options` - - :statuscode 200: no error - :statuscode 404: requested origin can not be found in the archive - - **Example:** - - .. parsed-literal:: - - :swh_web_api:`origin/1/visits/` """ result = {} - if origin_url: - origin_query = {'url': origin_url} - notfound_msg = 'No origin {} found'.format(origin_url) - url_args_next = {'origin_url': origin_url} - else: - origin_query = {'id': int(origin_id)} - notfound_msg = 'No origin {} found'.format(origin_id) - url_args_next = {'origin_id': origin_id} + origin_query = {'url': origin_url} + notfound_msg = 'No origin {} found'.format(origin_url) + url_args_next = {'origin_url': origin_url} per_page = int(request.query_params.get('per_page', '10')) last_visit = request.query_params.get('last_visit') if last_visit: @@ -525,11 +454,9 @@ @api_route(r'/origin/(?P.*)/visit/(?P[0-9]+)/', 'api-1-origin-visit') -@api_route(r'/origin/(?P[0-9]+)/visit/(?P[0-9]+)/', - 'api-1-origin-visit') @api_doc('/origin/visit/') @format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT) -def api_origin_visit(request, visit_id, origin_url=None, origin_id=None): +def api_origin_visit(request, visit_id, origin_url): """ .. http:get:: /api/1/origin/(origin_url)/visit/(visit_id)/ @@ -554,41 +481,7 @@ .. parsed-literal:: :swh_web_api:`origin/https://github.com/hylang/hy/visit/1/` - - .. http:get:: /api/1/origin/(origin_id)/visit/(visit_id)/ - - Get information about a specific visit of a software origin. - - .. warning:: - - All endpoints using an ``origin_id`` are deprecated and will be - removed in the near future. Only those using an ``origin_url`` - will remain available. - Use :http:get:`/api/1/origin/(origin_url)/visit/(visit_id)` - instead. - - :param int origin_id: a software origin identifier - :param int visit_id: a visit identifier - - {common_headers} - - {return_origin_visit} - - **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, - :http:method:`options` - - :statuscode 200: no error - :statuscode 404: requested origin or visit can not be found in the - archive - - **Example:** - - .. parsed-literal:: - - :swh_web_api:`origin/1500/visit/1/` """ - if not origin_url: - origin_url = service.lookup_origin({'id': int(origin_id)})['url'] return api_lookup( service.lookup_origin_visit, origin_url, int(visit_id), notfound_msg=('No visit {} for origin {} found' diff --git a/swh/web/api/views/revision.py b/swh/web/api/views/revision.py --- a/swh/web/api/views/revision.py +++ b/swh/web/api/views/revision.py @@ -7,7 +7,6 @@ from swh.web.common import service from swh.web.common.utils import reverse -from swh.web.common.utils import parse_timestamp from swh.web.api import utils from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route @@ -74,226 +73,6 @@ return result -@api_route(r'/revision/origin/(?P[0-9]+)' - r'/branch/(?P.+)/log/', - 'api-1-revision-origin-log') -@api_route(r'/revision/origin/(?P[0-9]+)/log/', - 'api-1-revision-origin-log') -@api_route(r'/revision/origin/(?P[0-9]+)' - r'/ts/(?P.+)/log/', - 'api-1-revision-origin-log') -@api_route(r'/revision/origin/(?P[0-9]+)' - r'/branch/(?P.+)' - r'/ts/(?P.+)/log/', - 'api-1-revision-origin-log') -@api_doc('/revision/origin/log/') -@format_docstring(return_revision_array=DOC_RETURN_REVISION_ARRAY) -def api_revision_log_by(request, origin_id, - branch_name='HEAD', - ts=None): - """ - .. http:get:: /api/1/revision/origin/(origin_id)[/branch/(branch_name)][/ts/(timestamp)]/log - - Show the commit log for a revision, searching for it based on software origin, - branch name, and/or visit timestamp. - - This endpoint behaves like :http:get:`/api/1/revision/(sha1_git)[/prev/(prev_sha1s)]/log/`, - but operates on the revision that has been found at a given software origin, - close to a given point in time, pointed by a given branch. - - .. warning:: - - All endpoints using an ``origin_id`` are deprecated and will be - removed in the near future. Only those using an ``origin_url`` - will remain available. - You should instead use successively - :http:get:`/api/1/origin/(origin_url)/visits/`, - :http:get:`/api/1/snapshot/(snapshot_id)/`, and - :http:get:`/api/1/revision/(sha1_git)[/prev/(prev_sha1s)]/log/`. - - - :param int origin_id: a software origin identifier - :param string branch_name: optional parameter specifying a fully-qualified branch name - associated to the software origin, e.g., "refs/heads/master". Defaults to the HEAD branch. - :param string timestamp: optional parameter specifying a timestamp close to which the revision - pointed by the given branch should be looked up. The timestamp can be expressed either - as an ISO date or as a Unix one (in UTC). Defaults to now. - - {common_headers} - - {return_revision_array} - - **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, - :http:method:`options` - - :statuscode 200: no error - :statuscode 404: no revision matching the given criteria could be found in the archive - - **Example:** - - .. parsed-literal:: - - :swh_web_api:`revision/origin/723566/ts/2016-01-17T00:00:00+00:00/log/` - """ # noqa - result = {} - per_page = int(request.query_params.get('per_page', '10')) - - def lookup_revision_log_by_with_limit(o_id, br, ts, limit=per_page+1): - return service.lookup_revision_log_by(o_id, br, ts, limit) - - error_msg = 'No revision matching origin %s ' % origin_id - error_msg += ', branch name %s' % branch_name - error_msg += (' and time stamp %s.' % ts) if ts else '.' - - rev_get = api_lookup( - lookup_revision_log_by_with_limit, int(origin_id), branch_name, ts, - notfound_msg=error_msg, - enrich_fn=utils.enrich_revision) - - nb_rev = len(rev_get) - if nb_rev == per_page+1: - revisions = rev_get[:-1] - last_sha1_git = rev_get[-1]['id'] - - params = {k: v for k, v in {'origin_id': origin_id, - 'branch_name': branch_name, - 'ts': ts, - }.items() if v is not None} - - query_params = {} - query_params['sha1_git'] = last_sha1_git - - if request.query_params.get('per_page'): - query_params['per_page'] = per_page - - result['headers'] = { - 'link-next': reverse('api-1-revision-origin-log', url_args=params, - query_params=query_params) - } - - else: - revisions = rev_get - - result.update({'results': revisions}) - - return result - - -@api_route(r'/revision/origin/(?P[0-9]+)/directory/', - 'api-1-revision-origin-directory') -@api_route(r'/revision/origin/(?P[0-9]+)/directory/(?P.+)/', - 'api-1-revision-origin-directory') -@api_route(r'/revision/origin/(?P[0-9]+)' - r'/branch/(?P.+)/directory/', - 'api-1-revision-origin-directory') -@api_route(r'/revision/origin/(?P[0-9]+)' - r'/branch/(?P.+)/ts/(?P.+)/directory/', - 'api-1-revision-origin-directory') -@api_route(r'/revision/origin/(?P[0-9]+)' - r'/branch/(?P.+)/directory/(?P.+)/', - 'api-1-revision-origin-directory') -@api_route(r'/revision/origin/(?P[0-9]+)' - r'/branch/(?P.+)/ts/(?P.+)' - r'/directory/(?P.+)/', - 'api-1-revision-origin-directory') -@api_doc('/revision/origin/directory/', tags=['hidden']) -def api_directory_through_revision_origin(request, origin_id, - branch_name='HEAD', - ts=None, - path=None, - with_data=False): - """ - Display directory or content information through a revision identified - by origin/branch/timestamp. - - .. warning:: - - All endpoints using an ``origin_id`` are deprecated and will be - removed in the near future. Only those using an ``origin_url`` - will remain available. - You should instead use successively - :http:get:`/api/1/origin/(origin_url)/visits/`, - :http:get:`/api/1/snapshot/(snapshot_id)/`, - :http:get:`/api/1/revision/(sha1_git)/`, - :http:get:`/api/1/directory/(sha1_git)/[(path)/]` - """ - if ts: - ts = parse_timestamp(ts) - - return _revision_directory_by({'origin_id': int(origin_id), - 'branch_name': branch_name, - 'ts': ts - }, - path, request.path, - with_data=with_data) - - -@api_route(r'/revision/origin/(?P[0-9]+)/', - 'api-1-revision-origin') -@api_route(r'/revision/origin/(?P[0-9]+)' - r'/branch/(?P.+)/', - 'api-1-revision-origin') -@api_route(r'/revision/origin/(?P[0-9]+)' - r'/branch/(?P.+)/ts/(?P.+)/', - 'api-1-revision-origin') -@api_route(r'/revision/origin/(?P[0-9]+)/ts/(?P.+)/', - 'api-1-revision-origin') -@api_doc('/revision/origin/') -@format_docstring(return_revision=DOC_RETURN_REVISION) -def api_revision_with_origin(request, origin_id, - branch_name='HEAD', - ts=None): - """ - .. http:get:: /api/1/revision/origin/(origin_id)/[branch/(branch_name)/][ts/(timestamp)/] - - Get information about a revision, searching for it based on software origin, - branch name, and/or visit timestamp. - - This endpoint behaves like :http:get:`/api/1/revision/(sha1_git)/`, - but operates on the revision that has been found at a given software origin, - close to a given point in time, pointed by a given branch. - - .. warning:: - - All endpoints using an ``origin_id`` are deprecated and will be - removed in the near future. Only those using an ``origin_url`` - will remain available. - You should instead use successively - :http:get:`/api/1/origin/(origin_url)/visits/`, - :http:get:`/api/1/snapshot/(snapshot_id)/`, and - :http:get:`/api/1/revision/(sha1_git)/`. - - :param int origin_id: a software origin identifier - :param string branch_name: optional parameter specifying a fully-qualified branch name - associated to the software origin, e.g., "refs/heads/master". Defaults to the HEAD branch. - :param string timestamp: optional parameter specifying a timestamp close to which the revision - pointed by the given branch should be looked up. The timestamp can be expressed either - as an ISO date or as a Unix one (in UTC). Defaults to now. - - {common_headers} - - {return_revision} - - **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, - :http:method:`options` - - :statuscode 200: no error - :statuscode 404: no revision matching the given criteria could be found in the archive - - **Example:** - - .. parsed-literal:: - - :swh_web_api:`revision/origin/13706355/branch/refs/heads/2.7/` - """ # noqa - return api_lookup( - service.lookup_revision_by, int(origin_id), branch_name, ts, - notfound_msg=('Revision with (origin_id: {}, branch_name: {}' - ', ts: {}) not found.'.format(origin_id, - branch_name, ts)), - enrich_fn=utils.enrich_revision) - - @api_route(r'/revision/(?P[0-9a-f]+)/', 'api-1-revision', checksum_args=['sha1_git']) @api_doc('/revision/') diff --git a/swh/web/common/service.py b/swh/web/common/service.py --- a/swh/web/common/service.py +++ b/swh/web/common/service.py @@ -525,7 +525,7 @@ if not rev_id: raise NotFoundExc('Revision for origin %s and branch %s not found.' - % (origin.get('url') or origin['id'], branch_name)) + % (origin.get('url'), branch_name)) return rev_id @@ -957,8 +957,6 @@ Args: revision: Dictionary of criterion to lookup the revision with. Here are the supported combination of possible values: - - origin_id, branch_name, ts, sha1_git - - origin_id, branch_name, ts - origin_url, branch_name, ts, sha1_git - origin_url, branch_name, ts - sha1_git_root, sha1_git @@ -979,16 +977,6 @@ revision['sha1_git'], limit) if ( - 'origin_id' in revision and - 'branch_name' in revision and - 'ts' in revision and - 'sha1_git' in revision): - return lookup_revision_with_context_by(revision['origin_id'], - revision['branch_name'], - revision['ts'], - revision['sha1_git'], - limit) - if ( 'origin_url' in revision and 'branch_name' in revision and 'ts' in revision): @@ -996,13 +984,6 @@ revision['branch_name'], revision['ts']) if ( - 'origin_id' in revision and - 'branch_name' in revision and - 'ts' in revision): - return lookup_revision_by(revision['origin_id'], - revision['branch_name'], - revision['ts']) - if ( 'sha1_git_root' in revision and 'sha1_git' in revision): return lookup_revision_with_context(revision['sha1_git_root'], diff --git a/swh/web/tests/api/views/test_origin.py b/swh/web/tests/api/views/test_origin.py --- a/swh/web/tests/api/views/test_origin.py +++ b/swh/web/tests/api/views/test_origin.py @@ -3,10 +3,7 @@ # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -import random - from hypothesis import given -import pytest from rest_framework.test import APITestCase from unittest.mock import patch @@ -15,7 +12,7 @@ from swh.web.common.utils import reverse from swh.web.common.origin_visits import get_origin_visits from swh.web.tests.strategies import ( - origin, new_origin, new_origins, visit_dates, new_snapshots + origin, new_origin, visit_dates, new_snapshots ) from swh.web.tests.testcase import WebTestCase @@ -84,13 +81,13 @@ def test_api_lookup_origin_visits(self, new_origin, visit_dates, new_snapshots): - origin_id = self.storage.origin_add_one(new_origin) - new_origin['id'] = origin_id + self.storage.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): - origin_visit = self.storage.origin_visit_add(origin_id, visit_date) + origin_visit = self.storage.origin_visit_add( + new_origin['url'], visit_date) self.storage.snapshot_add([new_snapshots[i]]) self.storage.origin_visit_update( - origin_id, origin_visit['visit'], + new_origin['url'], origin_visit['visit'], snapshot=new_snapshots[i]['id']) all_visits = list(reversed(get_origin_visits(new_origin))) @@ -127,13 +124,13 @@ def test_api_lookup_origin_visits_by_id(self, new_origin, visit_dates, new_snapshots): - origin_id = self.storage.origin_add_one(new_origin) - new_origin['id'] = origin_id + self.storage.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): - origin_visit = self.storage.origin_visit_add(origin_id, visit_date) + origin_visit = self.storage.origin_visit_add( + new_origin['url'], visit_date) self.storage.snapshot_add([new_snapshots[i]]) self.storage.origin_visit_update( - origin_id, origin_visit['visit'], + new_origin['url'], origin_visit['visit'], snapshot=new_snapshots[i]['id']) all_visits = list(reversed(get_origin_visits(new_origin))) @@ -170,14 +167,14 @@ def test_api_lookup_origin_visit(self, new_origin, visit_dates, new_snapshots): - origin_id = self.storage.origin_add_one(new_origin) - new_origin['id'] = origin_id + self.storage.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): - origin_visit = self.storage.origin_visit_add(origin_id, visit_date) + origin_visit = self.storage.origin_visit_add( + new_origin['url'], visit_date) visit_id = origin_visit['visit'] self.storage.snapshot_add([new_snapshots[i]]) self.storage.origin_visit_update( - origin_id, origin_visit['visit'], + new_origin['url'], origin_visit['visit'], snapshot=new_snapshots[i]['id']) url = reverse('api-1-origin-visit', url_args={'origin_url': new_origin['url'], @@ -187,7 +184,8 @@ self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') - expected_visit = self.origin_visit_get_by(origin_id, visit_id) + expected_visit = self.origin_visit_get_by( + new_origin['url'], visit_id) origin_url = reverse('api-1-origin', url_args={'origin_url': new_origin['url']}) @@ -205,17 +203,17 @@ def test_api_lookup_origin_visit_latest( self, new_origin, visit_dates, new_snapshots): - origin_id = self.storage.origin_add_one(new_origin) - new_origin['id'] = origin_id + self.storage.origin_add_one(new_origin) visit_dates.sort() visit_ids = [] for i, visit_date in enumerate(visit_dates): - origin_visit = self.storage.origin_visit_add(origin_id, visit_date) + origin_visit = self.storage.origin_visit_add( + new_origin['url'], visit_date) visit_ids.append(origin_visit['visit']) self.storage.snapshot_add([new_snapshots[0]]) self.storage.origin_visit_update( - origin_id, visit_ids[0], + new_origin['url'], visit_ids[0], snapshot=new_snapshots[0]['id']) url = reverse('api-1-origin-visit-latest', @@ -225,7 +223,8 @@ self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') - expected_visit = self.origin_visit_get_by(origin_id, visit_ids[1]) + expected_visit = self.origin_visit_get_by( + new_origin['url'], visit_ids[1]) origin_url = reverse('api-1-origin', url_args={'origin_url': new_origin['url']}) @@ -239,17 +238,17 @@ @given(new_origin(), visit_dates(2), new_snapshots(1)) def test_api_lookup_origin_visit_latest_with_snapshot( self, new_origin, visit_dates, new_snapshots): - origin_id = self.storage.origin_add_one(new_origin) - new_origin['id'] = origin_id + self.storage.origin_add_one(new_origin) visit_dates.sort() visit_ids = [] for i, visit_date in enumerate(visit_dates): - origin_visit = self.storage.origin_visit_add(origin_id, visit_date) + origin_visit = self.storage.origin_visit_add( + new_origin['url'], visit_date) visit_ids.append(origin_visit['visit']) self.storage.snapshot_add([new_snapshots[0]]) self.storage.origin_visit_update( - origin_id, visit_ids[0], + new_origin['url'], visit_ids[0], snapshot=new_snapshots[0]['id']) url = reverse('api-1-origin-visit-latest', @@ -260,7 +259,8 @@ self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') - expected_visit = self.origin_visit_get_by(origin_id, visit_ids[0]) + expected_visit = self.origin_visit_get_by( + new_origin['url'], visit_ids[0]) origin_url = reverse('api-1-origin', url_args={'origin_url': new_origin['url']}) @@ -274,42 +274,6 @@ self.assertEqual(rv.data, expected_visit) - @pytest.mark.origin_id - @given(new_origin(), visit_dates(3), new_snapshots(3)) - def test_api_lookup_origin_visit_by_id(self, new_origin, visit_dates, - new_snapshots): - - origin_id = self.storage.origin_add_one(new_origin) - new_origin['id'] = origin_id - for i, visit_date in enumerate(visit_dates): - origin_visit = self.storage.origin_visit_add(origin_id, visit_date) - visit_id = origin_visit['visit'] - self.storage.snapshot_add([new_snapshots[i]]) - self.storage.origin_visit_update( - origin_id, origin_visit['visit'], - snapshot=new_snapshots[i]['id']) - url = reverse('api-1-origin-visit', - url_args={'origin_id': origin_id, - 'visit_id': visit_id}) - - rv = self.client.get(url) - self.assertEqual(rv.status_code, 200, rv.data) - self.assertEqual(rv['Content-Type'], 'application/json') - - expected_visit = self.origin_visit_get_by(origin_id, visit_id) - - origin_url = reverse('api-1-origin', - url_args={'origin_url': new_origin['url']}) - snapshot_url = reverse( - 'api-1-snapshot', - url_args={'snapshot_id': expected_visit['snapshot']}) - - expected_visit['origin'] = new_origin['url'] - expected_visit['origin_url'] = origin_url - expected_visit['snapshot_url'] = snapshot_url - - self.assertEqual(rv.data, expected_visit) - @given(origin()) def test_api_lookup_origin_visit_not_found(self, origin): @@ -331,47 +295,6 @@ (origin['url'], max_visit_id+1) }) - @pytest.mark.origin_id - @given(origin()) - def test_api_lookup_origin_visit_not_found_by_id(self, origin): - - all_visits = list(reversed(get_origin_visits(origin))) - - max_visit_id = max([v['visit'] for v in all_visits]) - - url = reverse('api-1-origin-visit', - url_args={'origin_id': origin['id'], - 'visit_id': max_visit_id + 1}) - - rv = self.client.get(url) - - self.assertEqual(rv.status_code, 404, rv.data) - self.assertEqual(rv['Content-Type'], 'application/json') - self.assertEqual(rv.data, { - 'exception': 'NotFoundExc', - 'reason': 'Origin %s or its visit with id %s not found!' % - (origin['url'], max_visit_id+1) - }) - - @pytest.mark.origin_id - @given(origin()) - def test_api_origin_by_id(self, origin): - - url = reverse('api-1-origin', url_args={'origin_id': origin['id']}) - - rv = self.client.get(url) - - expected_origin = self.origin_get(origin) - - origin_visits_url = reverse('api-1-origin-visits', - url_args={'origin_url': origin['url']}) - - expected_origin['origin_visits_url'] = origin_visits_url - - self.assertEqual(rv.status_code, 200, rv.data) - self.assertEqual(rv['Content-Type'], 'application/json') - self.assertEqual(rv.data, expected_origin) - @given(origin()) def test_api_origin_by_url(self, origin): @@ -477,65 +400,8 @@ mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .assert_called_with(conjunction=['Jane Doe'], limit=70) - @pytest.mark.origin_id - @given(origin()) - def test_api_origin_metadata_search_missing_url(self, origin): - """indexer-storage with outdated db will return origin_url: None.""" - with patch('swh.web.common.service.idx_storage') as mock_idx_storage: - mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ - .side_effect = lambda conjunction, limit: [{ - 'id': origin['id'], - 'from_revision': ( - b'p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed ' - b'\xf2U\xfa\x05B8'), - 'metadata': {'author': 'Jane Doe'}, - 'origin_url': None, - 'tool': { - 'configuration': { - 'context': ['NpmMapping', 'CodemetaMapping'], - 'type': 'local' - }, - 'id': 3, - 'name': 'swh-metadata-detector', - 'version': '0.0.1' - } - }] - - url = reverse('api-1-origin-metadata-search', - query_params={'fulltext': 'Jane Doe'}) - rv = self.client.get(url) - - self.assertEqual(rv.status_code, 200, rv.content) - self.assertEqual(rv['Content-Type'], 'application/json') - expected_data = [{ - 'type': origin['type'], - 'url': origin['url'], - 'metadata': { - 'metadata': {'author': 'Jane Doe'}, - 'from_revision': ( - '7026b7c1a2af56521e951c01ed20f255fa054238'), - 'tool': { - 'configuration': { - 'context': ['NpmMapping', 'CodemetaMapping'], - 'type': 'local' - }, - 'id': 3, - 'name': 'swh-metadata-detector', - 'version': '0.0.1', - } - } - }] - actual_data = rv.data - for d in actual_data: - if 'id' in d: - del d['id'] - self.assertEqual(rv.data, expected_data) - mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ - .assert_called_with(conjunction=['Jane Doe'], limit=70) - @given(origin()) def test_api_origin_metadata_search_limit(self, origin): - with patch('swh.web.common.service.idx_storage') as mock_idx_storage: mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .side_effect = lambda conjunction, limit: [{ @@ -591,7 +457,7 @@ def test_api_origin_intrinsic_metadata(self, origin): with patch('swh.web.common.service.idx_storage') as mock_idx_storage: mock_idx_storage.origin_intrinsic_metadata_get \ - .side_effect = lambda origin_ids: [{ + .side_effect = lambda origin_urls: [{ 'from_revision': ( b'p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed ' b'\xf2U\xfa\x05B8'), @@ -628,47 +494,3 @@ self.assertEqual(rv.status_code, 400, rv.content) mock_idx_storage.assert_not_called() - - # in-memory storage does not handle different origin types for the - # same origin url but hypothesis will generate such kind of examples - @pytest.mark.xfail - @pytest.mark.origin_id - @given(new_origins(10)) - def test_api_lookup_origins(self, new_origins): - - nb_origins = len(new_origins) - - expected_origins = self.storage.origin_add(new_origins) - expected_origins.sort(key=lambda orig: orig['id']) - - origin_from_idx = random.randint(1, nb_origins-1) - 1 - origin_from = expected_origins[origin_from_idx]['id'] - max_origin_id = expected_origins[-1]['id'] - origin_count = random.randint(1, max_origin_id - origin_from) - - url = reverse('api-1-origins', - query_params={'origin_from': origin_from, - 'origin_count': origin_count}) - - rv = self.client.get(url) - - self.assertEqual(rv.status_code, 200, rv.data) - - start = origin_from_idx - end = origin_from_idx + origin_count - expected_origins = expected_origins[start:end] - - for expected_origin in expected_origins: - expected_origin['origin_visits_url'] = reverse( - 'api-1-origin-visits', - url_args={'origin_url': expected_origin['url']}) - - self.assertEqual(rv.data, expected_origins) - - next_origin_id = expected_origins[-1]['id']+1 - if self.storage.origin_get({'id': next_origin_id}): - self.assertIn('Link', rv) - next_url = reverse('api-1-origins', - query_params={'origin_from': next_origin_id, - 'origin_count': origin_count}) - self.assertIn(next_url, rv['Link']) diff --git a/swh/web/tests/api/views/test_revision.py b/swh/web/tests/api/views/test_revision.py --- a/swh/web/tests/api/views/test_revision.py +++ b/swh/web/tests/api/views/test_revision.py @@ -3,20 +3,15 @@ # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -import random - from hypothesis import given -import pytest from rest_framework.test import APITestCase from unittest.mock import patch from swh.web.common.exc import NotFoundExc -from swh.web.common.utils import reverse, parse_timestamp +from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 -from swh.web.tests.strategies import ( - revision, origin, origin_with_multiple_visits -) +from swh.web.tests.strategies import revision from swh.web.tests.testcase import WebTestCase @@ -77,193 +72,6 @@ 'reason': 'Revision with sha1_git %s not found.' % unknown_revision_}) - @pytest.mark.origin_id - def test_api_revision_with_origin_id_not_found(self): - unknown_origin_id = random.randint(1000, 1000000) - - url = reverse('api-1-revision-origin', - url_args={'origin_id': unknown_origin_id}) - rv = self.client.get(url) - - self.assertEqual(rv.status_code, 404, rv.data) - self.assertEqual(rv['content-type'], 'application/json') - self.assertEqual(rv.data, { - 'exception': 'NotFoundExc', - 'reason': 'Origin %s not found!' % - unknown_origin_id}) - - @pytest.mark.origin_id - @given(origin()) - def test_api_revision_with_origin_id(self, origin): - - url = reverse('api-1-revision-origin', - url_args={'origin_id': origin['id']}) - rv = self.client.get(url) - - snapshot = self.snapshot_get_latest(origin['url']) - expected_revision = self.revision_get( - snapshot['branches']['HEAD']['target']) - - self._enrich_revision(expected_revision) - - self.assertEqual(rv.status_code, 200, rv.data) - self.assertEqual(rv['Content-Type'], 'application/json') - self.assertEqual(rv.data, expected_revision) - - @pytest.mark.origin_id - @given(origin()) - def test_api_revision_with_origin_id_and_branch_name(self, origin): - - snapshot = self.snapshot_get_latest(origin['url']) - - branch_name = random.choice( - list(b for b in snapshot['branches'].keys() - if snapshot['branches'][b]['target_type'] == 'revision')) - - url = reverse('api-1-revision-origin', - url_args={'origin_id': origin['id'], - 'branch_name': branch_name}) - - rv = self.client.get(url) - - expected_revision = self.revision_get( - snapshot['branches'][branch_name]['target']) - - self._enrich_revision(expected_revision) - - self.assertEqual(rv.status_code, 200, rv.data) - self.assertEqual(rv['content-type'], 'application/json') - self.assertEqual(rv.data, expected_revision) - - @pytest.mark.origin_id - @given(origin_with_multiple_visits()) - def test_api_revision_with_origin_id_and_branch_name_and_ts(self, origin): - - visit = random.choice(self.origin_visit_get(origin['url'])) - - snapshot = self.snapshot_get(visit['snapshot']) - - branch_name = random.choice( - list(b for b in snapshot['branches'].keys() - if snapshot['branches'][b]['target_type'] == 'revision')) - - url = reverse('api-1-revision-origin', - url_args={'origin_id': origin['id'], - 'branch_name': branch_name, - 'ts': visit['date']}) - - rv = self.client.get(url) - - expected_revision = self.revision_get( - snapshot['branches'][branch_name]['target']) - - self._enrich_revision(expected_revision) - - self.assertEqual(rv.status_code, 200, rv.data) - self.assertEqual(rv['Content-Type'], 'application/json') - self.assertEqual(rv.data, expected_revision) - - @pytest.mark.origin_id - @given(origin_with_multiple_visits()) - def test_api_revision_with_origin_id_and_branch_name_and_ts_escapes( - self, origin): - visit = random.choice(self.origin_visit_get(origin['url'])) - - snapshot = self.snapshot_get(visit['snapshot']) - - branch_name = random.choice( - list(b for b in snapshot['branches'].keys() - if snapshot['branches'][b]['target_type'] == 'revision')) - - date = parse_timestamp(visit['date']) - - formatted_date = date.strftime('Today is %B %d, %Y at %X') - - url = reverse('api-1-revision-origin', - url_args={'origin_id': origin['id'], - 'branch_name': branch_name, - 'ts': formatted_date}) - - rv = self.client.get(url) - - expected_revision = self.revision_get( - snapshot['branches'][branch_name]['target']) - - self._enrich_revision(expected_revision) - - self.assertEqual(rv.status_code, 200, rv.data) - self.assertEqual(rv['Content-Type'], 'application/json') - self.assertEqual(rv.data, expected_revision) - - @pytest.mark.origin_id - def test_api_directory_through_revision_origin_id_ko(self): - unknown_origin_id_ = random.randint(1000, 1000000) - - url = reverse('api-1-revision-origin-directory', - url_args={'origin_id': unknown_origin_id_}) - rv = self.client.get(url) - - self.assertEqual(rv.status_code, 404, rv.data) - self.assertEqual(rv['Content-Type'], 'application/json') - self.assertEqual(rv.data, { - 'exception': 'NotFoundExc', - 'reason': 'Origin %s not found!' % - unknown_origin_id_ - }) - - @pytest.mark.origin_id - @given(origin()) - def test_api_directory_through_revision_origin_id(self, origin): - - url = reverse('api-1-revision-origin-directory', - url_args={'origin_id': origin['id']}) - rv = self.client.get(url) - - snapshot = self.snapshot_get_latest(origin['id']) - revision_id = snapshot['branches']['HEAD']['target'] - revision = self.revision_get(revision_id) - directory = self.directory_ls(revision['directory']) - - for entry in directory: - if entry['type'] == 'dir': - entry['target_url'] = reverse( - 'api-1-directory', - url_args={'sha1_git': entry['target']} - ) - entry['dir_url'] = reverse( - 'api-1-revision-origin-directory', - url_args={'origin_id': origin['id'], - 'path': entry['name']}) - elif entry['type'] == 'file': - entry['target_url'] = reverse( - 'api-1-content', - url_args={'q': 'sha1_git:%s' % entry['target']} - ) - entry['file_url'] = reverse( - 'api-1-revision-origin-directory', - url_args={'origin_id': origin['id'], - 'path': entry['name']}) - elif entry['type'] == 'rev': - entry['target_url'] = reverse( - 'api-1-revision', - url_args={'sha1_git': entry['target']} - ) - entry['rev_url'] = reverse( - 'api-1-revision-origin-directory', - url_args={'origin_id': origin['id'], - 'path': entry['name']}) - - expected_result = { - 'content': directory, - 'path': '.', - 'revision': revision_id, - 'type': 'dir' - } - - self.assertEqual(rv.status_code, 200, rv.data) - self.assertEqual(rv['Content-Type'], 'application/json') - self.assertEqual(rv.data, expected_result) - @given(revision()) def test_api_revision_log(self, revision): @@ -334,83 +142,6 @@ self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, expected_log) - @pytest.mark.origin_id - @given(origin()) - def test_api_revision_log_by_origin_id(self, origin): - - per_page = 10 - - url = reverse('api-1-revision-origin-log', - url_args={'origin_id': origin['id']}, - query_params={'per_page': per_page}) - - rv = self.client.get(url) - - snapshot = self.snapshot_get_latest(origin['url']) - - expected_log = self.revision_log( - snapshot['branches']['HEAD']['target'], limit=per_page+1) - - expected_log = list(map(self._enrich_revision, expected_log)) - - has_next = len(expected_log) > per_page - - self.assertEqual(rv.status_code, 200, rv.data) - self.assertEqual(rv['Content-Type'], 'application/json') - self.assertEqual(rv.data, - expected_log[:-1] if has_next else expected_log) - if has_next: - self.assertIn('Link', rv) - next_log_url = reverse( - 'api-1-revision-origin-log', - url_args={'origin_id': origin['id'], - 'branch_name': 'HEAD'}, - query_params={'per_page': per_page, - 'sha1_git': expected_log[-1]['id']}) - self.assertIn(next_log_url, rv['Link']) - - @pytest.mark.origin_id - @given(origin()) - def test_api_revision_log_by_ko(self, origin): - - invalid_branch_name = 'foobar' - - url = reverse('api-1-revision-origin-log', - url_args={'origin_id': origin['id'], - 'branch_name': invalid_branch_name}) - - rv = self.client.get(url) - - self.assertEqual(rv.status_code, 404, rv.data) - self.assertEqual(rv['Content-Type'], 'application/json') - self.assertFalse(rv.has_header('Link')) - self.assertEqual( - rv.data, - {'exception': 'NotFoundExc', - 'reason': 'Revision for origin %s and branch %s not found.' % - (origin['id'], invalid_branch_name)}) - - @pytest.mark.origin_id - @given(origin()) - def test_api_revision_log_by_origin_id_ko(self, origin): - - invalid_branch_name = 'foobar' - - url = reverse('api-1-revision-origin-log', - url_args={'origin_id': origin['id'], - 'branch_name': invalid_branch_name}) - - rv = self.client.get(url) - - self.assertEqual(rv.status_code, 404, rv.data) - self.assertEqual(rv['Content-Type'], 'application/json') - self.assertFalse(rv.has_header('Link')) - self.assertEqual( - rv.data, - {'exception': 'NotFoundExc', - 'reason': 'Revision for origin %s and branch %s not found.' % - (origin['id'], invalid_branch_name)}) - @patch('swh.web.api.views.revision._revision_directory_by') def test_api_revision_directory_ko_not_found(self, mock_rev_dir): # given diff --git a/swh/web/tests/common/test_service.py b/swh/web/tests/common/test_service.py --- a/swh/web/tests/common/test_service.py +++ b/swh/web/tests/common/test_service.py @@ -22,7 +22,7 @@ release, revision, unknown_revision, revisions, ancestor_revisions, non_ancestor_revisions, invalid_sha1, sha256, revision_with_submodules, empty_directory, - new_revision, new_origins + new_revision ) from swh.web.tests.testcase import ( WebTestCase, ctags_json_missing, fossology_missing @@ -188,43 +188,39 @@ @given(new_origin(), visit_dates()) def test_lookup_origin_visits(self, new_origin, visit_dates): - origin_id = self.storage.origin_add_one(new_origin) + self.storage.origin_add_one(new_origin) for ts in visit_dates: - self.storage.origin_visit_add(origin_id, ts) + self.storage.origin_visit_add(new_origin['url'], ts) actual_origin_visits = list( - service.lookup_origin_visits(origin_id, per_page=100)) + service.lookup_origin_visits(new_origin['url'], per_page=100)) - expected_visits = self.origin_visit_get(origin_id) + expected_visits = self.origin_visit_get(new_origin['url']) + for expected_visit in expected_visits: + expected_visit['origin'] = new_origin['url'] self.assertEqual(actual_origin_visits, expected_visits) @given(new_origin(), visit_dates()) def test_lookup_origin_visit(self, new_origin, visit_dates): - origin_id = self.storage.origin_add_one(new_origin) + self.storage.origin_add_one(new_origin) visits = [] for ts in visit_dates: - visits.append(self.storage.origin_visit_add(origin_id, ts)) + visits.append(self.storage.origin_visit_add( + new_origin['url'], ts)) visit = random.choice(visits)['visit'] - actual_origin_visit = service.lookup_origin_visit(origin_id, visit) + actual_origin_visit = service.lookup_origin_visit( + new_origin['url'], visit) - expected_visit = dict(self.storage.origin_visit_get_by(origin_id, - visit)) + expected_visit = dict(self.storage.origin_visit_get_by( + new_origin['url'], visit)) expected_visit['date'] = expected_visit['date'].isoformat() expected_visit['metadata'] = {} + expected_visit['origin'] = new_origin['url'] self.assertEqual(actual_origin_visit, expected_visit) - @pytest.mark.origin_id - @given(new_origin()) - def test_lookup_origin_by_id(self, new_origin): - origin_id = self.storage.origin_add_one(new_origin) - - actual_origin = service.lookup_origin({'id': origin_id}) - expected_origin = self.storage.origin_get({'id': origin_id}) - self.assertEqual(actual_origin, expected_origin) - @given(new_origin()) def test_lookup_origin(self, new_origin): self.storage.origin_add_one(new_origin) @@ -840,23 +836,3 @@ service.lookup_directory_with_revision( revision, dir_entry['name'], with_data=True)) ) - - @pytest.mark.origin_id - @given(new_origins(20)) - def test_lookup_origins(self, new_origins): - - nb_origins = len(new_origins) - expected_origins = self.storage.origin_add(new_origins) - expected_origins.sort(key=lambda orig: orig['id']) - - origin_from_idx = random.randint(1, nb_origins-1) - 1 - origin_from = expected_origins[origin_from_idx]['id'] - max_origin_idx = expected_origins[-1]['id'] - origin_count = random.randint(1, max_origin_idx - origin_from) - - actual_origins = list(service.lookup_origins(origin_from, - origin_count)) - expected_origins = list(self.storage.origin_get_range(origin_from, - origin_count)) - - self.assertEqual(actual_origins, expected_origins) diff --git a/swh/web/tests/strategies.py b/swh/web/tests/strategies.py --- a/swh/web/tests/strategies.py +++ b/swh/web/tests/strategies.py @@ -11,7 +11,7 @@ from hypothesis import settings, assume from hypothesis.strategies import ( just, sampled_from, lists, composite, datetimes, - integers, binary, text, characters + binary, text, characters ) from swh.model.hashutil import hash_to_hex, hash_to_bytes @@ -233,14 +233,6 @@ return sampled_from(ret) -def unknown_origin_id(): - """ - Hypothesis strategy returning a random origin id not ingested - into the test archive. - """ - return integers(min_value=1000000) - - def new_origin(): """ Hypothesis strategy returning a random origin not ingested diff --git a/swh/web/tests/testcase.py b/swh/web/tests/testcase.py --- a/swh/web/tests/testcase.py +++ b/swh/web/tests/testcase.py @@ -121,20 +121,20 @@ return list(map(converters.from_revision, self.storage.revision_log([rev_id_bytes], limit=limit))) - def snapshot_get_latest(self, origin_id): - snp = self.storage.snapshot_get_latest(origin_id) + def snapshot_get_latest(self, origin_url): + snp = self.storage.snapshot_get_latest(origin_url) return converters.from_snapshot(snp) def origin_get(self, origin_info): origin = self.storage.origin_get(origin_info) return converters.from_origin(origin) - def origin_visit_get(self, origin_id): - visits = self.storage.origin_visit_get(origin_id) + def origin_visit_get(self, origin_url): + visits = self.storage.origin_visit_get(origin_url) return list(map(converters.from_origin_visit, visits)) - def origin_visit_get_by(self, origin_id, visit_id): - visit = self.storage.origin_visit_get_by(origin_id, visit_id) + def origin_visit_get_by(self, origin_url, visit_id): + visit = self.storage.origin_visit_get_by(origin_url, visit_id) return converters.from_origin_visit(visit) def snapshot_get(self, snapshot_id): diff --git a/tox.ini b/tox.ini --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ setenv = SWH_STORAGE_IN_MEMORY_ENABLE_ORIGIN_IDS=false commands = - pytest --hypothesis-profile=swh-web-fast --cov {envsitepackagesdir}/swh/web --cov-branch {posargs} {envsitepackagesdir}/swh/web -m "not origin_id" + pytest --hypothesis-profile=swh-web-fast --cov {envsitepackagesdir}/swh/web --cov-branch {posargs} {envsitepackagesdir}/swh/web [testenv:py3-slow] deps =