Differential D7959 Diff 28721 swh/web/api/views/origin.py

Changeset View

Standalone View

swh/web/api/views/origin.py

# Copyright (C) 2015-2020 The Software Heritage developers		# Copyright (C) 2015-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version		# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from distutils.util import strtobool		from distutils.util import strtobool
from functools import partial		from functools import partial
		from typing import Dict

		from rest_framework.request import Request

from swh.search.exc import SearchQuerySyntaxError		from swh.search.exc import SearchQuerySyntaxError
from swh.web.api.apidoc import api_doc, format_docstring		from swh.web.api.apidoc import api_doc, format_docstring
from swh.web.api.apiurls import api_route		from swh.web.api.apiurls import api_route
from swh.web.api.utils import (		from swh.web.api.utils import (
enrich_origin,		enrich_origin,
enrich_origin_search_result,		enrich_origin_search_result,
enrich_origin_visit,		enrich_origin_visit,
Show All 35 Lines	DOC_RETURN_ORIGIN_VISIT_ARRAY += """
:http:get:`/api/1/origin/(origin_url)/visit/(visit_id)/`		:http:get:`/api/1/origin/(origin_url)/visit/(visit_id)/`
in order to get information about the visit		in order to get information about the visit
"""		"""


@api_route(r"/origins/", "api-1-origins")		@api_route(r"/origins/", "api-1-origins")
@api_doc("/origins/", noargs=True)		@api_doc("/origins/", noargs=True)
@format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY)		@format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY)
def api_origins(request):		def api_origins(request: Request):
"""		"""
.. http:get:: /api/1/origins/		.. http:get:: /api/1/origins/

Get list of archived software origins.		Get list of archived software origins.

.. warning::		.. warning::

This endpoint used to provide an ``origin_from`` query parameter,		This endpoint used to provide an ``origin_from`` query parameter,
Show All 22 Lines	def api_origins(request: Request):

if old_param_origin_from:		if old_param_origin_from:
raise BadInputExc("Please use the Link header to browse through result")		raise BadInputExc("Please use the Link header to browse through result")

page_token = request.query_params.get("page_token", None)		page_token = request.query_params.get("page_token", None)
limit = min(int(request.query_params.get("origin_count", "100")), 10000)		limit = min(int(request.query_params.get("origin_count", "100")), 10000)

page_result = archive.lookup_origins(page_token, limit)		page_result = archive.lookup_origins(page_token, limit)
origins = [enrich_origin(o, request=request) for o in page_result.results]		origins = [enrich_origin(o, request=request) for o in page_result.results]
		vlorentzUnsubmitted Not Done Inline Actions Instead of this change, change the type of `enrich_origin` to take `Union[Dict[str, Any], OriginInfo]` as argument vlorentz: Instead of this change, change the type of `enrich_origin` to take `Union[Dict[str, Any]…
next_page_token = page_result.next_page_token		next_page_token = page_result.next_page_token

response = {"results": origins, "headers": {}}		headers: Dict[str, str] = {}
if next_page_token is not None:		if next_page_token is not None:
response["headers"]["link-next"] = reverse(		headers["link-next"] = reverse(
"api-1-origins",		"api-1-origins",
query_params={"page_token": next_page_token, "origin_count": limit},		query_params={"page_token": next_page_token, "origin_count": limit},
request=request,		request=request,
)		)
return response		return {"results": origins, "headers": headers}


@api_route(r"/origin/(?P<origin_url>.+)/get/", "api-1-origin")		@api_route(r"/origin/(?P<origin_url>.+)/get/", "api-1-origin")
@api_doc("/origin/")		@api_doc("/origin/")
@format_docstring(return_origin=DOC_RETURN_ORIGIN)		@format_docstring(return_origin=DOC_RETURN_ORIGIN)
def api_origin(request, origin_url):		def api_origin(request: Request, origin_url: str):
"""		"""
.. http:get:: /api/1/origin/(origin_url)/get/		.. http:get:: /api/1/origin/(origin_url)/get/

Get information about a software origin.		Get information about a software origin.

:param string origin_url: the origin url		:param string origin_url: the origin url

{return_origin}		{return_origin}
Show All 18 Lines	return api_lookup(
archive.lookup_origin,		archive.lookup_origin,
ori_dict,		ori_dict,
notfound_msg=error_msg,		notfound_msg=error_msg,
enrich_fn=enrich_origin,		enrich_fn=enrich_origin,
request=request,		request=request,
)		)


def _visit_types():		def _visit_types() -> str:
docstring = ""		docstring = ""
# available visit types are queried using swh-search so we do it in a try		# available visit types are queried using swh-search so we do it in a try
# block in case of failure (for instance in docker environment when		# block in case of failure (for instance in docker environment when
# elasticsearch service is not available)		# elasticsearch service is not available)
try:		try:
visit_types = [f"{visit_type}" for visit_type in origin_visit_types()]		visit_types = [f"{visit_type}" for visit_type in origin_visit_types()]
docstring = ", ".join(visit_types[:-1]) + f", and {visit_types[-1]}"		docstring = ", ".join(visit_types[:-1]) + f", and {visit_types[-1]}"
except Exception:		except Exception:
docstring = "???"		docstring = "???"
pass		pass
return docstring		return docstring


@api_route(		@api_route(
r"/origin/search/(?P<url_pattern>.*)/",		r"/origin/search/(?P<url_pattern>.*)/",
"api-1-origin-search",		"api-1-origin-search",
throttle_scope="swh_api_origin_search",		throttle_scope="swh_api_origin_search",
)		)
@api_doc("/origin/search/")		@api_doc("/origin/search/")
@format_docstring(		@format_docstring(
return_origin_array=DOC_RETURN_ORIGIN_ARRAY, visit_types=_visit_types()		return_origin_array=DOC_RETURN_ORIGIN_ARRAY, visit_types=_visit_types()
)		)
def api_origin_search(request, url_pattern):		def api_origin_search(request: Request, url_pattern: str):
"""		"""
.. http:get:: /api/1/origin/search/(url_pattern)/		.. http:get:: /api/1/origin/search/(url_pattern)/

Search for software origins whose urls contain a provided string		Search for software origins whose urls contain a provided string
pattern or match a provided regular expression.		pattern or match a provided regular expression.
The search is performed in a case insensitive way.		The search is performed in a case insensitive way.

.. warning::		.. warning::
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines	def api_origin_search(request: Request, url_pattern: str):
result.update({"results": results})		result.update({"results": results})

return result		return result


@api_route(r"/origin/metadata-search/", "api-1-origin-metadata-search")		@api_route(r"/origin/metadata-search/", "api-1-origin-metadata-search")
@api_doc("/origin/metadata-search/", noargs=True)		@api_doc("/origin/metadata-search/", noargs=True)
@format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY)		@format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY)
def api_origin_metadata_search(request):		def api_origin_metadata_search(request: Request):
"""		"""
.. http:get:: /api/1/origin/metadata-search/		.. http:get:: /api/1/origin/metadata-search/

Search for software origins whose metadata (expressed as a		Search for software origins whose metadata (expressed as a
JSON-LD/CodeMeta dictionary) match the provided criteria.		JSON-LD/CodeMeta dictionary) match the provided criteria.
For now, only full-text search on this dictionary is supported.		For now, only full-text search on this dictionary is supported.

:query str fulltext: a string that will be matched against origin		:query str fulltext: a string that will be matched against origin
Show All 24 Lines	results = api_lookup(
archive.search_origin_metadata, fulltext, limit, request=request		archive.search_origin_metadata, fulltext, limit, request=request
)		)

return {		return {
"results": results,		"results": results,
}		}


@api_route(r"/origin/(?P<origin_url>.*)/visits/", "api-1-origin-visits")		@api_route(r"/origin/(?P<origin_url>.+)/visits/", "api-1-origin-visits")
@api_doc("/origin/visits/")		@api_doc("/origin/visits/")
@format_docstring(return_origin_visit_array=DOC_RETURN_ORIGIN_VISIT_ARRAY)		@format_docstring(return_origin_visit_array=DOC_RETURN_ORIGIN_VISIT_ARRAY)
def api_origin_visits(request, origin_url):		def api_origin_visits(request: Request, origin_url: str):
"""		"""
.. http:get:: /api/1/origin/(origin_url)/visits/		.. http:get:: /api/1/origin/(origin_url)/visits/

Get information about all visits of a software origin.		Get information about all visits of a software origin.
Visits are returned sorted in descending order according		Visits are returned sorted in descending order according
to their date.		to their date.

:param str origin_url: a software origin URL		:param str origin_url: a software origin URL
Show All 17 Lines	.. http:get:: /api/1/origin/(origin_url)/visits/
:swh_web_api:`origin/https://github.com/hylang/hy/visits/`		:swh_web_api:`origin/https://github.com/hylang/hy/visits/`

"""		"""
result = {}		result = {}
origin_query = {"url": origin_url}		origin_query = {"url": origin_url}
notfound_msg = "No origin {} found".format(origin_url)		notfound_msg = "No origin {} found".format(origin_url)
url_args_next = {"origin_url": origin_url}		url_args_next = {"origin_url": origin_url}
per_page = int(request.query_params.get("per_page", "10"))		per_page = int(request.query_params.get("per_page", "10"))
last_visit = request.query_params.get("last_visit")		last_visit_str = request.query_params.get("last_visit")
if last_visit:		last_visit = int(last_visit_str) if last_visit_str else None
last_visit = int(last_visit)

def _lookup_origin_visits(origin_query, last_visit=last_visit, per_page=per_page):		def _lookup_origin_visits(origin_query, last_visit=last_visit, per_page=per_page):
all_visits = get_origin_visits(origin_query)		all_visits = get_origin_visits(origin_query)
all_visits.reverse()		all_visits.reverse()
visits = []		visits = []
if not last_visit:		if not last_visit:
visits = all_visits[:per_page]		visits = all_visits[:per_page]
else:		else:
Show All 34 Lines	if results:
}		}

result.update({"results": results})		result.update({"results": results})

return result		return result


@api_route(		@api_route(
r"/origin/(?P<origin_url>.*)/visit/latest/",		r"/origin/(?P<origin_url>.+)/visit/latest/",
"api-1-origin-visit-latest",		"api-1-origin-visit-latest",
throttle_scope="swh_api_origin_visit_latest",		throttle_scope="swh_api_origin_visit_latest",
)		)
@api_doc("/origin/visit/latest/")		@api_doc("/origin/visit/latest/")
@format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT)		@format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT)
def api_origin_visit_latest(request, origin_url=None):		def api_origin_visit_latest(request: Request, origin_url: str):
"""		"""
.. http:get:: /api/1/origin/(origin_url)/visit/latest/		.. http:get:: /api/1/origin/(origin_url)/visit/latest/

Get information about the latest visit of a software origin.		Get information about the latest visit of a software origin.

:param str origin_url: a software origin URL		:param str origin_url: a software origin URL
:query boolean require_snapshot: if true, only return a visit		:query boolean require_snapshot: if true, only return a visit
with a snapshot		with a snapshot
Show All 21 Lines	return api_lookup(
enrich_fn=partial(		enrich_fn=partial(
enrich_origin_visit, with_origin_link=True, with_origin_visit_link=False		enrich_origin_visit, with_origin_link=True, with_origin_visit_link=False
),		),
request=request,		request=request,
)		)


@api_route(		@api_route(
r"/origin/(?P<origin_url>.*)/visit/(?P<visit_id>[0-9]+)/", "api-1-origin-visit"		r"/origin/(?P<origin_url>.+)/visit/(?P<visit_id>[0-9]+)/", "api-1-origin-visit"
)		)
@api_doc("/origin/visit/")		@api_doc("/origin/visit/")
@format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT)		@format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT)
def api_origin_visit(request, visit_id, origin_url):		def api_origin_visit(request: Request, visit_id: str, origin_url: str):
"""		"""
.. http:get:: /api/1/origin/(origin_url)/visit/(visit_id)/		.. http:get:: /api/1/origin/(origin_url)/visit/(visit_id)/

Get information about a specific visit of a software origin.		Get information about a specific visit of a software origin.

:param str origin_url: a software origin URL		:param str origin_url: a software origin URL
:param int visit_id: a visit identifier		:param int visit_id: a visit identifier

Show All 23 Lines	def api_origin_visit(request: Request, visit_id: str, origin_url: str):
)		)


@api_route(		@api_route(
r"/origin/(?P<origin_url>.+)/intrinsic-metadata/", "api-origin-intrinsic-metadata"		r"/origin/(?P<origin_url>.+)/intrinsic-metadata/", "api-origin-intrinsic-metadata"
)		)
@api_doc("/origin/intrinsic-metadata/")		@api_doc("/origin/intrinsic-metadata/")
@format_docstring()		@format_docstring()
def api_origin_intrinsic_metadata(request, origin_url):		def api_origin_intrinsic_metadata(request: Request, origin_url: str):
"""		"""
.. http:get:: /api/1/origin/(origin_url)/intrinsic-metadata		.. http:get:: /api/1/origin/(origin_url)/intrinsic-metadata

Get intrinsic metadata of a software origin (as a JSON-LD/CodeMeta dictionary).		Get intrinsic metadata of a software origin (as a JSON-LD/CodeMeta dictionary).

:param string origin_url: the origin url		:param string origin_url: the origin url

:>json string ???: intrinsic metadata field of the origin		:>json string ???: intrinsic metadata field of the origin
Show All 19 Lines