Differential D2492 Diff 8849 swh/web/api/views/origin.py

Changeset View

Standalone View

swh/web/api/views/origin.py

# Copyright (C) 2015-2019 The Software Heritage developers		# Copyright (C) 2015-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version		# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from distutils.util import strtobool		from distutils.util import strtobool
from functools import partial		from functools import partial

from swh.web.common import service		from swh.web.common import service
from swh.web.common.exc import BadInputExc		from swh.web.common.exc import BadInputExc
from swh.web.common.origin_visits import get_origin_visits		from swh.web.common.origin_visits import get_origin_visits
from swh.web.common.utils import reverse		from swh.web.common.utils import reverse
from swh.web.api.apidoc import api_doc, format_docstring		from swh.web.api.apidoc import api_doc, format_docstring
from swh.web.api.apiurls import api_route		from swh.web.api.apiurls import api_route
		from swh.web.api.utils import enrich_origin, enrich_origin_visit
from swh.web.api.views.utils import api_lookup		from swh.web.api.views.utils import api_lookup


DOC_RETURN_ORIGIN = '''		DOC_RETURN_ORIGIN = '''
:>json string origin_visits_url: link to in order to get information		:>json string origin_visits_url: link to in order to get information
about the visits for that origin		about the visits for that origin
:>json string url: the origin canonical url		:>json string url: the origin canonical url
:>json string type: the type of software origin (deprecated value;		:>json string type: the type of software origin (deprecated value;
Show All 26 Lines
DOC_RETURN_ORIGIN_VISIT_ARRAY += '''		DOC_RETURN_ORIGIN_VISIT_ARRAY += '''
:>jsonarr number id: the unique identifier of the origin		:>jsonarr number id: the unique identifier of the origin
:>jsonarr string origin_visit_url: link to		:>jsonarr string origin_visit_url: link to
:http:get:`/api/1/origin/(origin_url)/visit/(visit_id)/`		:http:get:`/api/1/origin/(origin_url)/visit/(visit_id)/`
in order to get information about the visit		in order to get information about the visit
'''		'''


def _enrich_origin(origin):
if 'url' in origin:
o = origin.copy()
o['origin_visits_url'] = reverse(
'api-1-origin-visits', url_args={'origin_url': origin['url']})
return o

return origin


def _enrich_origin_visit(origin_visit, *,
with_origin_link, with_origin_visit_link):
ov = origin_visit.copy()
if with_origin_link:
ov['origin_url'] = reverse('api-1-origin',
url_args={'origin_url': ov['origin']})
if with_origin_visit_link:
ov['origin_visit_url'] = reverse('api-1-origin-visit',
url_args={'origin_url': ov['origin'],
'visit_id': ov['visit']})
snapshot = ov['snapshot']
if snapshot:
ov['snapshot_url'] = reverse('api-1-snapshot',
url_args={'snapshot_id': snapshot})
else:
ov['snapshot_url'] = None
return ov


@api_route(r'/origins/', 'api-1-origins')		@api_route(r'/origins/', 'api-1-origins')
@api_doc('/origins/', noargs=True)		@api_doc('/origins/', noargs=True)
@format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY)		@format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY)
def api_origins(request):		def api_origins(request):
"""		"""
.. http:get:: /api/1/origins/		.. http:get:: /api/1/origins/

Get list of archived software origins.		Get list of archived software origins.
Show All 24 Lines	.. http:get:: /api/1/origins/

:swh_web_api:`origins?origin_count=500`		:swh_web_api:`origins?origin_count=500`
"""		"""
origin_from = int(request.query_params.get('origin_from', '1'))		origin_from = int(request.query_params.get('origin_from', '1'))
origin_count = int(request.query_params.get('origin_count', '100'))		origin_count = int(request.query_params.get('origin_count', '100'))
origin_count = min(origin_count, 10000)		origin_count = min(origin_count, 10000)
results = api_lookup(		results = api_lookup(
service.lookup_origins, origin_from, origin_count+1,		service.lookup_origins, origin_from, origin_count+1,
enrich_fn=_enrich_origin)		enrich_fn=enrich_origin,
		request=request)
response = {'results': results, 'headers': {}}		response = {'results': results, 'headers': {}}
if len(results) > origin_count:		if len(results) > origin_count:
origin_from = results.pop()['id']		origin_from = results.pop()['id']
response['headers']['link-next'] = reverse(		response['headers']['link-next'] = reverse(
'api-1-origins',		'api-1-origins',
query_params={'origin_from': origin_from,		query_params={'origin_from': origin_from,
'origin_count': origin_count})		'origin_count': origin_count},
		request=request)
return response		return response


@api_route(r'/origin/(?P<origin_url>.+)/get/', 'api-1-origin')		@api_route(r'/origin/(?P<origin_url>.+)/get/', 'api-1-origin')
@api_doc('/origin/')		@api_doc('/origin/')
@format_docstring(return_origin=DOC_RETURN_ORIGIN)		@format_docstring(return_origin=DOC_RETURN_ORIGIN)
def api_origin(request, origin_url):		def api_origin(request, origin_url):
"""		"""
Show All 24 Lines	ori_dict = {
'url': origin_url		'url': origin_url
}		}

error_msg = 'Origin with url %s not found.' % ori_dict['url']		error_msg = 'Origin with url %s not found.' % ori_dict['url']

return api_lookup(		return api_lookup(
service.lookup_origin, ori_dict,		service.lookup_origin, ori_dict,
notfound_msg=error_msg,		notfound_msg=error_msg,
enrich_fn=_enrich_origin)		enrich_fn=enrich_origin,
		request=request)


@api_route(r'/origin/search/(?P<url_pattern>.+)/',		@api_route(r'/origin/search/(?P<url_pattern>.+)/',
'api-1-origin-search',		'api-1-origin-search',
throttle_scope='swh_api_origin_search')		throttle_scope='swh_api_origin_search')
@api_doc('/origin/search/')		@api_doc('/origin/search/')
@format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY)		@format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY)
def api_origin_search(request, url_pattern):		def api_origin_search(request, url_pattern):
Show All 38 Lines	def api_origin_search(request, url_pattern):
result = {}		result = {}
offset = int(request.query_params.get('offset', '0'))		offset = int(request.query_params.get('offset', '0'))
limit = min(int(request.query_params.get('limit', '70')), 1000)		limit = min(int(request.query_params.get('limit', '70')), 1000)
regexp = request.query_params.get('regexp', 'false')		regexp = request.query_params.get('regexp', 'false')
with_visit = request.query_params.get('with_visit', 'false')		with_visit = request.query_params.get('with_visit', 'false')

results = api_lookup(service.search_origin, url_pattern, offset, limit,		results = api_lookup(service.search_origin, url_pattern, offset, limit,
bool(strtobool(regexp)), bool(strtobool(with_visit)),		bool(strtobool(regexp)), bool(strtobool(with_visit)),
enrich_fn=_enrich_origin)		enrich_fn=enrich_origin,
		request=request)

nb_results = len(results)		nb_results = len(results)
if nb_results == limit:		if nb_results == limit:
query_params = {}		query_params = {}
query_params['offset'] = offset + limit		query_params['offset'] = offset + limit
query_params['limit'] = limit		query_params['limit'] = limit
query_params['regexp'] = regexp		query_params['regexp'] = regexp

result['headers'] = {		result['headers'] = {
'link-next': reverse('api-1-origin-search',		'link-next': reverse('api-1-origin-search',
url_args={'url_pattern': url_pattern},		url_args={'url_pattern': url_pattern},
query_params=query_params)		query_params=query_params,
		request=request)
}		}

result.update({		result.update({
'results': results		'results': results
})		})

return result		return result

Show All 33 Lines	def api_origin_metadata_search(request):
"""		"""
fulltext = request.query_params.get('fulltext', None)		fulltext = request.query_params.get('fulltext', None)
limit = min(int(request.query_params.get('limit', '70')), 100)		limit = min(int(request.query_params.get('limit', '70')), 100)

if not fulltext:		if not fulltext:
content = '"fulltext" must be provided and non-empty.'		content = '"fulltext" must be provided and non-empty.'
raise BadInputExc(content)		raise BadInputExc(content)

results = api_lookup(service.search_origin_metadata, fulltext, limit)		results = api_lookup(service.search_origin_metadata, fulltext, limit,
		request=request)

return {		return {
'results': results,		'results': results,
}		}


@api_route(r'/origin/(?P<origin_url>.*)/visits/', 'api-1-origin-visits')		@api_route(r'/origin/(?P<origin_url>.*)/visits/', 'api-1-origin-visits')
@api_doc('/origin/visits/')		@api_doc('/origin/visits/')
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines	def _lookup_origin_visits(
if v['visit'] == last_visit:		if v['visit'] == last_visit:
visits = all_visits[i+1:i+1+per_page]		visits = all_visits[i+1:i+1+per_page]
break		break
for v in visits:		for v in visits:
yield v		yield v

results = api_lookup(_lookup_origin_visits, origin_query,		results = api_lookup(_lookup_origin_visits, origin_query,
notfound_msg=notfound_msg,		notfound_msg=notfound_msg,
enrich_fn=partial(_enrich_origin_visit,		enrich_fn=partial(enrich_origin_visit,
with_origin_link=False,		with_origin_link=False,
with_origin_visit_link=True))		with_origin_visit_link=True),
		request=request)

if results:		if results:
nb_results = len(results)		nb_results = len(results)
if nb_results == per_page:		if nb_results == per_page:
new_last_visit = results[-1]['visit']		new_last_visit = results[-1]['visit']
query_params = {}		query_params = {}
query_params['last_visit'] = new_last_visit		query_params['last_visit'] = new_last_visit

if request.query_params.get('per_page'):		if request.query_params.get('per_page'):
query_params['per_page'] = per_page		query_params['per_page'] = per_page

result['headers'] = {		result['headers'] = {
'link-next': reverse('api-1-origin-visits',		'link-next': reverse('api-1-origin-visits',
url_args=url_args_next,		url_args=url_args_next,
query_params=query_params)		query_params=query_params,
		request=request)
}		}

result.update({		result.update({
'results': results		'results': results
})		})

return result		return result

Show All 31 Lines	.. http:get:: /api/1/origin/(origin_url)/visit/latest/
:swh_web_api:`origin/https://github.com/hylang/hy/visit/latest/`		:swh_web_api:`origin/https://github.com/hylang/hy/visit/latest/`
"""		"""
require_snapshot = request.query_params.get('require_snapshot', 'false')		require_snapshot = request.query_params.get('require_snapshot', 'false')
return api_lookup(		return api_lookup(
service.lookup_origin_visit_latest, origin_url,		service.lookup_origin_visit_latest, origin_url,
bool(strtobool(require_snapshot)),		bool(strtobool(require_snapshot)),
notfound_msg=('No visit for origin {} found'		notfound_msg=('No visit for origin {} found'
.format(origin_url)),		.format(origin_url)),
enrich_fn=partial(_enrich_origin_visit,		enrich_fn=partial(enrich_origin_visit,
with_origin_link=True,		with_origin_link=True,
with_origin_visit_link=False))		with_origin_visit_link=False),
		request=request)


@api_route(r'/origin/(?P<origin_url>.*)/visit/(?P<visit_id>[0-9]+)/',		@api_route(r'/origin/(?P<origin_url>.*)/visit/(?P<visit_id>[0-9]+)/',
'api-1-origin-visit')		'api-1-origin-visit')
@api_doc('/origin/visit/')		@api_doc('/origin/visit/')
@format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT)		@format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT)
def api_origin_visit(request, visit_id, origin_url):		def api_origin_visit(request, visit_id, origin_url):
"""		"""
Show All 20 Lines	.. http:get:: /api/1/origin/(origin_url)/visit/(visit_id)/
.. parsed-literal::		.. parsed-literal::

:swh_web_api:`origin/https://github.com/hylang/hy/visit/1/`		:swh_web_api:`origin/https://github.com/hylang/hy/visit/1/`
"""		"""
return api_lookup(		return api_lookup(
service.lookup_origin_visit, origin_url, int(visit_id),		service.lookup_origin_visit, origin_url, int(visit_id),
notfound_msg=('No visit {} for origin {} found'		notfound_msg=('No visit {} for origin {} found'
.format(visit_id, origin_url)),		.format(visit_id, origin_url)),
enrich_fn=partial(_enrich_origin_visit,		enrich_fn=partial(enrich_origin_visit,
with_origin_link=True,		with_origin_link=True,
with_origin_visit_link=False))		with_origin_visit_link=False),
		request=request)


@api_route(r'/origin/(?P<origin_url>.+)'		@api_route(r'/origin/(?P<origin_url>.+)'
'/intrinsic-metadata', 'api-origin-intrinsic-metadata')		'/intrinsic-metadata', 'api-origin-intrinsic-metadata')
@api_doc('/origin/intrinsic-metadata/')		@api_doc('/origin/intrinsic-metadata/')
@format_docstring()		@format_docstring()
def api_origin_intrinsic_metadata(request, origin_url):		def api_origin_intrinsic_metadata(request, origin_url):
"""		"""
Show All 23 Lines	ori_dict = {
'url': origin_url		'url': origin_url
}		}

error_msg = 'Origin with url %s not found' % ori_dict['url']		error_msg = 'Origin with url %s not found' % ori_dict['url']

return api_lookup(		return api_lookup(
service.lookup_origin_intrinsic_metadata, ori_dict,		service.lookup_origin_intrinsic_metadata, ori_dict,
notfound_msg=error_msg,		notfound_msg=error_msg,
enrich_fn=_enrich_origin)		enrich_fn=enrich_origin,
		request=request)