Changeset View
Changeset View
Standalone View
Standalone View
swh/web/api/views/origin.py
# Copyright (C) 2015-2019 The Software Heritage developers | # Copyright (C) 2015-2019 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU Affero General Public License version 3, or any later version | # License: GNU Affero General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from distutils.util import strtobool | from distutils.util import strtobool | ||||
from functools import partial | from functools import partial | ||||
from swh.web.common import service | from swh.web.common import service | ||||
from swh.web.common.exc import BadInputExc | from swh.web.common.exc import BadInputExc | ||||
from swh.web.common.origin_visits import get_origin_visits | from swh.web.common.origin_visits import get_origin_visits | ||||
from swh.web.common.utils import reverse | from swh.web.common.utils import reverse | ||||
from swh.web.api.apidoc import api_doc, format_docstring | from swh.web.api.apidoc import api_doc, format_docstring | ||||
from swh.web.api.apiurls import api_route | from swh.web.api.apiurls import api_route | ||||
from swh.web.api.utils import enrich_origin, enrich_origin_visit | |||||
from swh.web.api.views.utils import api_lookup | from swh.web.api.views.utils import api_lookup | ||||
DOC_RETURN_ORIGIN = ''' | DOC_RETURN_ORIGIN = ''' | ||||
:>json string origin_visits_url: link to in order to get information | :>json string origin_visits_url: link to in order to get information | ||||
about the visits for that origin | about the visits for that origin | ||||
:>json string url: the origin canonical url | :>json string url: the origin canonical url | ||||
:>json string type: the type of software origin (deprecated value; | :>json string type: the type of software origin (deprecated value; | ||||
Show All 26 Lines | |||||
DOC_RETURN_ORIGIN_VISIT_ARRAY += ''' | DOC_RETURN_ORIGIN_VISIT_ARRAY += ''' | ||||
:>jsonarr number id: the unique identifier of the origin | :>jsonarr number id: the unique identifier of the origin | ||||
:>jsonarr string origin_visit_url: link to | :>jsonarr string origin_visit_url: link to | ||||
:http:get:`/api/1/origin/(origin_url)/visit/(visit_id)/` | :http:get:`/api/1/origin/(origin_url)/visit/(visit_id)/` | ||||
in order to get information about the visit | in order to get information about the visit | ||||
''' | ''' | ||||
def _enrich_origin(origin): | |||||
if 'url' in origin: | |||||
o = origin.copy() | |||||
o['origin_visits_url'] = reverse( | |||||
'api-1-origin-visits', url_args={'origin_url': origin['url']}) | |||||
return o | |||||
return origin | |||||
def _enrich_origin_visit(origin_visit, *, | |||||
with_origin_link, with_origin_visit_link): | |||||
ov = origin_visit.copy() | |||||
if with_origin_link: | |||||
ov['origin_url'] = reverse('api-1-origin', | |||||
url_args={'origin_url': ov['origin']}) | |||||
if with_origin_visit_link: | |||||
ov['origin_visit_url'] = reverse('api-1-origin-visit', | |||||
url_args={'origin_url': ov['origin'], | |||||
'visit_id': ov['visit']}) | |||||
snapshot = ov['snapshot'] | |||||
if snapshot: | |||||
ov['snapshot_url'] = reverse('api-1-snapshot', | |||||
url_args={'snapshot_id': snapshot}) | |||||
else: | |||||
ov['snapshot_url'] = None | |||||
return ov | |||||
@api_route(r'/origins/', 'api-1-origins') | @api_route(r'/origins/', 'api-1-origins') | ||||
@api_doc('/origins/', noargs=True) | @api_doc('/origins/', noargs=True) | ||||
@format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY) | @format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY) | ||||
def api_origins(request): | def api_origins(request): | ||||
""" | """ | ||||
.. http:get:: /api/1/origins/ | .. http:get:: /api/1/origins/ | ||||
Get list of archived software origins. | Get list of archived software origins. | ||||
Show All 24 Lines | .. http:get:: /api/1/origins/ | ||||
:swh_web_api:`origins?origin_count=500` | :swh_web_api:`origins?origin_count=500` | ||||
""" | """ | ||||
origin_from = int(request.query_params.get('origin_from', '1')) | origin_from = int(request.query_params.get('origin_from', '1')) | ||||
origin_count = int(request.query_params.get('origin_count', '100')) | origin_count = int(request.query_params.get('origin_count', '100')) | ||||
origin_count = min(origin_count, 10000) | origin_count = min(origin_count, 10000) | ||||
results = api_lookup( | results = api_lookup( | ||||
service.lookup_origins, origin_from, origin_count+1, | service.lookup_origins, origin_from, origin_count+1, | ||||
enrich_fn=_enrich_origin) | enrich_fn=enrich_origin, | ||||
request=request) | |||||
response = {'results': results, 'headers': {}} | response = {'results': results, 'headers': {}} | ||||
if len(results) > origin_count: | if len(results) > origin_count: | ||||
origin_from = results.pop()['id'] | origin_from = results.pop()['id'] | ||||
response['headers']['link-next'] = reverse( | response['headers']['link-next'] = reverse( | ||||
'api-1-origins', | 'api-1-origins', | ||||
query_params={'origin_from': origin_from, | query_params={'origin_from': origin_from, | ||||
'origin_count': origin_count}) | 'origin_count': origin_count}, | ||||
request=request) | |||||
return response | return response | ||||
@api_route(r'/origin/(?P<origin_url>.+)/get/', 'api-1-origin') | @api_route(r'/origin/(?P<origin_url>.+)/get/', 'api-1-origin') | ||||
@api_doc('/origin/') | @api_doc('/origin/') | ||||
@format_docstring(return_origin=DOC_RETURN_ORIGIN) | @format_docstring(return_origin=DOC_RETURN_ORIGIN) | ||||
def api_origin(request, origin_url): | def api_origin(request, origin_url): | ||||
""" | """ | ||||
Show All 24 Lines | ori_dict = { | ||||
'url': origin_url | 'url': origin_url | ||||
} | } | ||||
error_msg = 'Origin with url %s not found.' % ori_dict['url'] | error_msg = 'Origin with url %s not found.' % ori_dict['url'] | ||||
return api_lookup( | return api_lookup( | ||||
service.lookup_origin, ori_dict, | service.lookup_origin, ori_dict, | ||||
notfound_msg=error_msg, | notfound_msg=error_msg, | ||||
enrich_fn=_enrich_origin) | enrich_fn=enrich_origin, | ||||
request=request) | |||||
@api_route(r'/origin/search/(?P<url_pattern>.+)/', | @api_route(r'/origin/search/(?P<url_pattern>.+)/', | ||||
'api-1-origin-search', | 'api-1-origin-search', | ||||
throttle_scope='swh_api_origin_search') | throttle_scope='swh_api_origin_search') | ||||
@api_doc('/origin/search/') | @api_doc('/origin/search/') | ||||
@format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY) | @format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY) | ||||
def api_origin_search(request, url_pattern): | def api_origin_search(request, url_pattern): | ||||
Show All 36 Lines | def api_origin_search(request, url_pattern): | ||||
result = {} | result = {} | ||||
limit = min(int(request.query_params.get('limit', '70')), 1000) | limit = min(int(request.query_params.get('limit', '70')), 1000) | ||||
page_token = request.query_params.get('page_token') | page_token = request.query_params.get('page_token') | ||||
with_visit = request.query_params.get('with_visit', 'false') | with_visit = request.query_params.get('with_visit', 'false') | ||||
(results, page_token) = api_lookup( | (results, page_token) = api_lookup( | ||||
service.search_origin, url_pattern, limit, | service.search_origin, url_pattern, limit, | ||||
bool(strtobool(with_visit)), page_token, | bool(strtobool(with_visit)), page_token, | ||||
enrich_fn=_enrich_origin) | enrich_fn=enrich_origin, request=request) | ||||
if page_token is not None: | if page_token is not None: | ||||
query_params = {} | query_params = {} | ||||
query_params['limit'] = limit | query_params['limit'] = limit | ||||
query_params['page_token'] = page_token | query_params['page_token'] = page_token | ||||
result['headers'] = { | result['headers'] = { | ||||
'link-next': reverse('api-1-origin-search', | 'link-next': reverse('api-1-origin-search', | ||||
url_args={'url_pattern': url_pattern}, | url_args={'url_pattern': url_pattern}, | ||||
query_params=query_params) | query_params=query_params, | ||||
request=request) | |||||
} | } | ||||
result.update({ | result.update({ | ||||
'results': results | 'results': results | ||||
}) | }) | ||||
return result | return result | ||||
Show All 33 Lines | def api_origin_metadata_search(request): | ||||
""" | """ | ||||
fulltext = request.query_params.get('fulltext', None) | fulltext = request.query_params.get('fulltext', None) | ||||
limit = min(int(request.query_params.get('limit', '70')), 100) | limit = min(int(request.query_params.get('limit', '70')), 100) | ||||
if not fulltext: | if not fulltext: | ||||
content = '"fulltext" must be provided and non-empty.' | content = '"fulltext" must be provided and non-empty.' | ||||
raise BadInputExc(content) | raise BadInputExc(content) | ||||
results = api_lookup(service.search_origin_metadata, fulltext, limit) | results = api_lookup(service.search_origin_metadata, fulltext, limit, | ||||
request=request) | |||||
return { | return { | ||||
'results': results, | 'results': results, | ||||
} | } | ||||
@api_route(r'/origin/(?P<origin_url>.*)/visits/', 'api-1-origin-visits') | @api_route(r'/origin/(?P<origin_url>.*)/visits/', 'api-1-origin-visits') | ||||
@api_doc('/origin/visits/') | @api_doc('/origin/visits/') | ||||
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | def _lookup_origin_visits( | ||||
if v['visit'] == last_visit: | if v['visit'] == last_visit: | ||||
visits = all_visits[i+1:i+1+per_page] | visits = all_visits[i+1:i+1+per_page] | ||||
break | break | ||||
for v in visits: | for v in visits: | ||||
yield v | yield v | ||||
results = api_lookup(_lookup_origin_visits, origin_query, | results = api_lookup(_lookup_origin_visits, origin_query, | ||||
notfound_msg=notfound_msg, | notfound_msg=notfound_msg, | ||||
enrich_fn=partial(_enrich_origin_visit, | enrich_fn=partial(enrich_origin_visit, | ||||
with_origin_link=False, | with_origin_link=False, | ||||
with_origin_visit_link=True)) | with_origin_visit_link=True), | ||||
request=request) | |||||
if results: | if results: | ||||
nb_results = len(results) | nb_results = len(results) | ||||
if nb_results == per_page: | if nb_results == per_page: | ||||
new_last_visit = results[-1]['visit'] | new_last_visit = results[-1]['visit'] | ||||
query_params = {} | query_params = {} | ||||
query_params['last_visit'] = new_last_visit | query_params['last_visit'] = new_last_visit | ||||
if request.query_params.get('per_page'): | if request.query_params.get('per_page'): | ||||
query_params['per_page'] = per_page | query_params['per_page'] = per_page | ||||
result['headers'] = { | result['headers'] = { | ||||
'link-next': reverse('api-1-origin-visits', | 'link-next': reverse('api-1-origin-visits', | ||||
url_args=url_args_next, | url_args=url_args_next, | ||||
query_params=query_params) | query_params=query_params, | ||||
request=request) | |||||
} | } | ||||
result.update({ | result.update({ | ||||
'results': results | 'results': results | ||||
}) | }) | ||||
return result | return result | ||||
Show All 31 Lines | .. http:get:: /api/1/origin/(origin_url)/visit/latest/ | ||||
:swh_web_api:`origin/https://github.com/hylang/hy/visit/latest/` | :swh_web_api:`origin/https://github.com/hylang/hy/visit/latest/` | ||||
""" | """ | ||||
require_snapshot = request.query_params.get('require_snapshot', 'false') | require_snapshot = request.query_params.get('require_snapshot', 'false') | ||||
return api_lookup( | return api_lookup( | ||||
service.lookup_origin_visit_latest, origin_url, | service.lookup_origin_visit_latest, origin_url, | ||||
bool(strtobool(require_snapshot)), | bool(strtobool(require_snapshot)), | ||||
notfound_msg=('No visit for origin {} found' | notfound_msg=('No visit for origin {} found' | ||||
.format(origin_url)), | .format(origin_url)), | ||||
enrich_fn=partial(_enrich_origin_visit, | enrich_fn=partial(enrich_origin_visit, | ||||
with_origin_link=True, | with_origin_link=True, | ||||
with_origin_visit_link=False)) | with_origin_visit_link=False), | ||||
request=request) | |||||
@api_route(r'/origin/(?P<origin_url>.*)/visit/(?P<visit_id>[0-9]+)/', | @api_route(r'/origin/(?P<origin_url>.*)/visit/(?P<visit_id>[0-9]+)/', | ||||
'api-1-origin-visit') | 'api-1-origin-visit') | ||||
@api_doc('/origin/visit/') | @api_doc('/origin/visit/') | ||||
@format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT) | @format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT) | ||||
def api_origin_visit(request, visit_id, origin_url): | def api_origin_visit(request, visit_id, origin_url): | ||||
""" | """ | ||||
Show All 20 Lines | .. http:get:: /api/1/origin/(origin_url)/visit/(visit_id)/ | ||||
.. parsed-literal:: | .. parsed-literal:: | ||||
:swh_web_api:`origin/https://github.com/hylang/hy/visit/1/` | :swh_web_api:`origin/https://github.com/hylang/hy/visit/1/` | ||||
""" | """ | ||||
return api_lookup( | return api_lookup( | ||||
service.lookup_origin_visit, origin_url, int(visit_id), | service.lookup_origin_visit, origin_url, int(visit_id), | ||||
notfound_msg=('No visit {} for origin {} found' | notfound_msg=('No visit {} for origin {} found' | ||||
.format(visit_id, origin_url)), | .format(visit_id, origin_url)), | ||||
enrich_fn=partial(_enrich_origin_visit, | enrich_fn=partial(enrich_origin_visit, | ||||
with_origin_link=True, | with_origin_link=True, | ||||
with_origin_visit_link=False)) | with_origin_visit_link=False), | ||||
request=request) | |||||
@api_route(r'/origin/(?P<origin_url>.+)' | @api_route(r'/origin/(?P<origin_url>.+)' | ||||
'/intrinsic-metadata', 'api-origin-intrinsic-metadata') | '/intrinsic-metadata', 'api-origin-intrinsic-metadata') | ||||
@api_doc('/origin/intrinsic-metadata/') | @api_doc('/origin/intrinsic-metadata/') | ||||
@format_docstring() | @format_docstring() | ||||
def api_origin_intrinsic_metadata(request, origin_url): | def api_origin_intrinsic_metadata(request, origin_url): | ||||
""" | """ | ||||
Show All 23 Lines | ori_dict = { | ||||
'url': origin_url | 'url': origin_url | ||||
} | } | ||||
error_msg = 'Origin with url %s not found' % ori_dict['url'] | error_msg = 'Origin with url %s not found' % ori_dict['url'] | ||||
return api_lookup( | return api_lookup( | ||||
service.lookup_origin_intrinsic_metadata, ori_dict, | service.lookup_origin_intrinsic_metadata, ori_dict, | ||||
notfound_msg=error_msg, | notfound_msg=error_msg, | ||||
enrich_fn=_enrich_origin) | enrich_fn=enrich_origin, | ||||
request=request) |