diff --git a/swh/web/api/utils.py b/swh/web/api/utils.py --- a/swh/web/api/utils.py +++ b/swh/web/api/utils.py @@ -1,13 +1,14 @@ -# Copyright (C) 2015-2020 The Software Heritage developers +# Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, Union from django.http import HttpRequest from swh.web.common.query import parse_hash +from swh.web.common.typing import OriginInfo from swh.web.common.utils import resolve_branch_alias, reverse @@ -136,9 +137,9 @@ def enrich_content( content: Dict[str, Any], + request: Optional[HttpRequest] = None, top_url: Optional[bool] = False, query_string: Optional[str] = None, - request: Optional[HttpRequest] = None, ) -> Dict[str, str]: """Enrich content with links to: - data_url: its raw data @@ -273,7 +274,7 @@ def enrich_origin( - origin: Dict[str, Any], request: Optional[HttpRequest] = None + origin: Union[Dict[str, Any], OriginInfo], request: Optional[HttpRequest] = None ) -> Dict[str, Any]: """Enrich origin dict with link to its visits @@ -284,14 +285,15 @@ Returns: An enriched origin dict filled with an additional url """ - if "url" in origin: - origin["origin_visits_url"] = reverse( + origin_dict = dict(origin) + if "url" in origin_dict: + origin_dict["origin_visits_url"] = reverse( "api-1-origin-visits", - url_args={"origin_url": origin["url"]}, + url_args={"origin_url": origin_dict["url"]}, request=request, ) - return origin + return origin_dict def enrich_origin_search_result( @@ -313,10 +315,9 @@ def enrich_origin_visit( origin_visit: Dict[str, Any], - *, - with_origin_link: bool, - with_origin_visit_link: bool, request: Optional[HttpRequest] = None, + with_origin_link: bool = False, + with_origin_visit_link: bool = False, ) -> Dict[str, Any]: """Enrich origin visit dict with additional links diff --git a/swh/web/api/views/content.py b/swh/web/api/views/content.py --- a/swh/web/api/views/content.py +++ b/swh/web/api/views/content.py @@ -1,11 +1,13 @@ -# Copyright (C) 2015-2019 The Software Heritage developers +# Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import functools +from typing import Optional from django.http import HttpResponse +from rest_framework.request import Request from swh.web.api import utils from swh.web.api.apidoc import api_doc, format_docstring @@ -23,7 +25,7 @@ ) @api_doc("/content/filetype/") @format_docstring() -def api_content_filetype(request, q): +def api_content_filetype(request: Request, q: str): """ .. http:get:: /api/1/content/[(hash_type):](hash)/filetype/ @@ -73,7 +75,7 @@ ) @api_doc("/content/language/") @format_docstring() -def api_content_language(request, q): +def api_content_language(request: Request, q: str): """ .. http:get:: /api/1/content/[(hash_type):](hash)/language/ @@ -124,7 +126,7 @@ ) @api_doc("/content/license/") @format_docstring() -def api_content_license(request, q): +def api_content_license(request: Request, q: str): """ .. http:get:: /api/1/content/[(hash_type):](hash)/license/ @@ -167,7 +169,7 @@ @api_route(r"/content/(?P[0-9a-z_:]*[0-9a-f]+)/ctags/", "api-1-content-ctags") @api_doc("/content/ctags/", tags=["hidden"]) -def api_content_ctags(request, q): +def api_content_ctags(request: Request, q: str): """ Get information about all `Ctags `_-style symbols defined in a content object. @@ -187,7 +189,7 @@ checksum_args=["q"], ) @api_doc("/content/raw/") -def api_content_raw(request, q): +def api_content_raw(request: Request, q: str): """ .. http:get:: /api/1/content/[(hash_type):](hash)/raw/ @@ -235,7 +237,7 @@ @api_route(r"/content/symbol/(?P.+)/", "api-1-content-symbol") @api_doc("/content/symbol/", tags=["hidden"]) -def api_content_symbol(request, q=None): +def api_content_symbol(request: Request, q: str): """Search content objects by `Ctags `_-style symbol (e.g., function name, data type, method, ...). @@ -281,10 +283,10 @@ @api_route(r"/content/known/search/", "api-1-content-known", methods=["POST"]) -@api_route(r"/content/known/(?P(?!search).*)/", "api-1-content-known") +@api_route(r"/content/known/(?P(?!search).+)/", "api-1-content-known") @api_doc("/content/known/", tags=["hidden"]) @format_docstring() -def api_check_content_known(request, q=None): +def api_check_content_known(request: Request, q: Optional[str] = None): """ .. http:get:: /api/1/content/known/(sha1)[,(sha1), ...,(sha1)]/ @@ -311,7 +313,6 @@ :swh_web_api:`content/known/dc2830a9e72f23c1dfebef4413003221baa5fb62,0c3f19cb47ebfbe643fb19fa94c874d18fa62d12/` """ - response = {"search_res": None, "search_stats": None} search_stats = {"nbfiles": 0, "pct": 0} search_res = None @@ -345,11 +346,9 @@ search_res = result nbfound = len([x for x in lookup if x["found"]]) search_stats["nbfiles"] = nb_queries - search_stats["pct"] = (nbfound / nb_queries) * 100 + search_stats["pct"] = int((nbfound / nb_queries) * 100) - response["search_res"] = search_res - response["search_stats"] = search_stats - return response + return {"search_res": search_res, "search_stats": search_stats} @api_route( @@ -357,7 +356,7 @@ ) @api_doc("/content/") @format_docstring() -def api_content_metadata(request, q): +def api_content_metadata(request: Request, q: str): """ .. http:get:: /api/1/content/[(hash_type):](hash)/ diff --git a/swh/web/api/views/directory.py b/swh/web/api/views/directory.py --- a/swh/web/api/views/directory.py +++ b/swh/web/api/views/directory.py @@ -1,8 +1,12 @@ -# Copyright (C) 2015-2019 The Software Heritage developers +# Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +from typing import Optional + +from rest_framework.request import Request + from swh.web.api import utils from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route @@ -22,7 +26,7 @@ ) @api_doc("/directory/") @format_docstring() -def api_directory(request, sha1_git, path=None): +def api_directory(request: Request, sha1_git: str, path: Optional[str] = None): """ .. http:get:: /api/1/directory/(sha1_git)/[(path)/] diff --git a/swh/web/api/views/identifiers.py b/swh/web/api/views/identifiers.py --- a/swh/web/api/views/identifiers.py +++ b/swh/web/api/views/identifiers.py @@ -1,9 +1,14 @@ -# Copyright (C) 2018-2021 The Software Heritage developers +# Copyright (C) 2018-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +from typing import Dict, Set + +from rest_framework.request import Request + from swh.model.hashutil import hash_to_bytes, hash_to_hex +from swh.model.swhids import ObjectType from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.common import archive @@ -11,10 +16,10 @@ from swh.web.common.identifiers import get_swhid, group_swhids, resolve_swhid -@api_route(r"/resolve/(?P.*)/", "api-1-resolve-swhid") +@api_route(r"/resolve/(?P.+)/", "api-1-resolve-swhid") @api_doc("/resolve/") @format_docstring() -def api_resolve_swhid(request, swhid): +def api_resolve_swhid(request: Request, swhid: str): """ .. http:get:: /api/1/resolve/(swhid)/ @@ -71,7 +76,7 @@ @api_route(r"/known/", "api-1-known", methods=["POST"]) @api_doc("/known/") @format_docstring() -def api_swhid_known(request): +def api_swhid_known(request: Request): """ .. http:post:: /api/1/known/ @@ -109,7 +114,7 @@ # group swhids by their type swhids_by_type = group_swhids(swhids) # search for hashes not present in the storage - missing_hashes = { + missing_hashes: Dict[ObjectType, Set[bytes]] = { k: set(map(hash_to_bytes, archive.lookup_missing_hashes({k: v}))) for k, v in swhids_by_type.items() } diff --git a/swh/web/api/views/metadata.py b/swh/web/api/views/metadata.py --- a/swh/web/api/views/metadata.py +++ b/swh/web/api/views/metadata.py @@ -1,14 +1,16 @@ -# Copyright (C) 2021 The Software Heritage developers +# Copyright (C) 2021-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import base64 import re +from typing import Dict, Optional import iso8601 from django.http import HttpResponse +from rest_framework.request import Request from swh.model import hashutil, swhids from swh.model.model import MetadataAuthority, MetadataAuthorityType @@ -25,7 +27,7 @@ ) @api_doc("/raw-extrinsic-metadata/swhid/") @format_docstring() -def api_raw_extrinsic_metadata_swhid(request, target): +def api_raw_extrinsic_metadata_swhid(request: Request, target: str): """ .. http:get:: /api/1/raw-extrinsic-metadata/swhid/(target) @@ -71,12 +73,12 @@ :swh_web_api:`raw-extrinsic-metadata/swhid/swh:1:dir:a2faa28028657859c16ff506924212b33f0e1307/?authority=forge%20https://pypi.org/` """ # noqa - authority_str: str = request.query_params.get("authority") - after_str: str = request.query_params.get("after") + authority_str: Optional[str] = request.query_params.get("authority") + after_str: Optional[str] = request.query_params.get("after") limit_str: str = request.query_params.get("limit", "100") - page_token_str: str = request.query_params.get("page_token") + page_token_str: Optional[str] = request.query_params.get("page_token") - if not authority_str: + if authority_str is None: raise BadInputExc("The 'authority' query parameter is required.") if " " not in authority_str.strip(): raise BadInputExc("The 'authority' query parameter should contain a space.") @@ -106,17 +108,17 @@ limit = min(limit, 10000) try: - target = swhids.CoreSWHID.from_string(target).to_extended() + parsed_target = swhids.CoreSWHID.from_string(target).to_extended() except swhids.ValidationError as e: raise BadInputExc(f"Invalid target SWHID: {e.args[0]}") from None - if page_token_str: + if page_token_str is not None: page_token = base64.urlsafe_b64decode(page_token_str) else: page_token = None result_page = archive.storage.raw_extrinsic_metadata_get( - target=target, + target=parsed_target, authority=authority, after=after, page_token=page_token, @@ -139,13 +141,9 @@ results.append(result) - response = { - "results": results, - "headers": {}, - } - + headers: Dict[str, str] = {} if result_page.next_page_token is not None: - response["headers"]["link-next"] = reverse( + headers["link-next"] = reverse( "api-1-raw-extrinsic-metadata-swhid", url_args={"target": target}, query_params=dict( @@ -159,14 +157,17 @@ request=request, ) - return response + return { + "results": results, + "headers": headers, + } @api_route( "/raw-extrinsic-metadata/get/(?P[0-9a-z]+)/", "api-1-raw-extrinsic-metadata-get", ) -def api_raw_extrinsic_metadata_get(request, id): +def api_raw_extrinsic_metadata_get(request: Request, id: str): # This is an internal endpoint that should only be accessed via URLs given # by /raw-extrinsic-metadata/swhid/; so it is not documented. metadata = archive.storage.raw_extrinsic_metadata_get_by_ids( @@ -198,7 +199,7 @@ ) @api_doc("/raw-extrinsic-metadata/swhid/authorities/") @format_docstring() -def api_raw_extrinsic_metadata_swhid_authorities(request, target): +def api_raw_extrinsic_metadata_swhid_authorities(request: Request, target: str): """ .. http:get:: /api/1/raw-extrinsic-metadata/swhid/(target)/authorities/ @@ -225,20 +226,21 @@ :swh_web_api:`raw-extrinsic-metadata/swhid/swh:1:dir:a2faa28028657859c16ff506924212b33f0e1307/authorities/` """ # noqa - target_str = target try: - target = swhids.CoreSWHID.from_string(target_str).to_extended() + parsed_target = swhids.CoreSWHID.from_string(target).to_extended() except swhids.ValidationError as e: raise BadInputExc(f"Invalid target SWHID: {e.args[0]}") from None - authorities = archive.storage.raw_extrinsic_metadata_get_authorities(target=target) + authorities = archive.storage.raw_extrinsic_metadata_get_authorities( + target=parsed_target + ) results = [ { **authority.to_dict(), "metadata_list_url": reverse( "api-1-raw-extrinsic-metadata-swhid", - url_args={"target": target_str}, + url_args={"target": target}, query_params={"authority": f"{authority.type.value} {authority.url}"}, request=request, ), diff --git a/swh/web/api/views/origin.py b/swh/web/api/views/origin.py --- a/swh/web/api/views/origin.py +++ b/swh/web/api/views/origin.py @@ -1,10 +1,13 @@ -# Copyright (C) 2015-2020 The Software Heritage developers +# Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from distutils.util import strtobool from functools import partial +from typing import Dict + +from rest_framework.request import Request from swh.search.exc import SearchQuerySyntaxError from swh.web.api.apidoc import api_doc, format_docstring @@ -56,7 +59,7 @@ @api_route(r"/origins/", "api-1-origins") @api_doc("/origins/", noargs=True) @format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY) -def api_origins(request): +def api_origins(request: Request): """ .. http:get:: /api/1/origins/ @@ -98,20 +101,20 @@ origins = [enrich_origin(o, request=request) for o in page_result.results] next_page_token = page_result.next_page_token - response = {"results": origins, "headers": {}} + headers: Dict[str, str] = {} if next_page_token is not None: - response["headers"]["link-next"] = reverse( + headers["link-next"] = reverse( "api-1-origins", query_params={"page_token": next_page_token, "origin_count": limit}, request=request, ) - return response + return {"results": origins, "headers": headers} @api_route(r"/origin/(?P.+)/get/", "api-1-origin") @api_doc("/origin/") @format_docstring(return_origin=DOC_RETURN_ORIGIN) -def api_origin(request, origin_url): +def api_origin(request: Request, origin_url: str): """ .. http:get:: /api/1/origin/(origin_url)/get/ @@ -146,7 +149,7 @@ ) -def _visit_types(): +def _visit_types() -> str: docstring = "" # available visit types are queried using swh-search so we do it in a try # block in case of failure (for instance in docker environment when @@ -169,7 +172,7 @@ @format_docstring( return_origin_array=DOC_RETURN_ORIGIN_ARRAY, visit_types=_visit_types() ) -def api_origin_search(request, url_pattern): +def api_origin_search(request: Request, url_pattern: str): """ .. http:get:: /api/1/origin/search/(url_pattern)/ @@ -249,7 +252,7 @@ @api_route(r"/origin/metadata-search/", "api-1-origin-metadata-search") @api_doc("/origin/metadata-search/", noargs=True) @format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY) -def api_origin_metadata_search(request): +def api_origin_metadata_search(request: Request): """ .. http:get:: /api/1/origin/metadata-search/ @@ -290,10 +293,10 @@ } -@api_route(r"/origin/(?P.*)/visits/", "api-1-origin-visits") +@api_route(r"/origin/(?P.+)/visits/", "api-1-origin-visits") @api_doc("/origin/visits/") @format_docstring(return_origin_visit_array=DOC_RETURN_ORIGIN_VISIT_ARRAY) -def api_origin_visits(request, origin_url): +def api_origin_visits(request: Request, origin_url: str): """ .. http:get:: /api/1/origin/(origin_url)/visits/ @@ -327,9 +330,8 @@ notfound_msg = "No origin {} found".format(origin_url) url_args_next = {"origin_url": origin_url} per_page = int(request.query_params.get("per_page", "10")) - last_visit = request.query_params.get("last_visit") - if last_visit: - last_visit = int(last_visit) + last_visit_str = request.query_params.get("last_visit") + last_visit = int(last_visit_str) if last_visit_str else None def _lookup_origin_visits(origin_query, last_visit=last_visit, per_page=per_page): all_visits = get_origin_visits(origin_query) @@ -380,13 +382,13 @@ @api_route( - r"/origin/(?P.*)/visit/latest/", + r"/origin/(?P.+)/visit/latest/", "api-1-origin-visit-latest", throttle_scope="swh_api_origin_visit_latest", ) @api_doc("/origin/visit/latest/") @format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT) -def api_origin_visit_latest(request, origin_url=None): +def api_origin_visit_latest(request: Request, origin_url: str): """ .. http:get:: /api/1/origin/(origin_url)/visit/latest/ @@ -424,11 +426,11 @@ @api_route( - r"/origin/(?P.*)/visit/(?P[0-9]+)/", "api-1-origin-visit" + r"/origin/(?P.+)/visit/(?P[0-9]+)/", "api-1-origin-visit" ) @api_doc("/origin/visit/") @format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT) -def api_origin_visit(request, visit_id, origin_url): +def api_origin_visit(request: Request, visit_id: str, origin_url: str): """ .. http:get:: /api/1/origin/(origin_url)/visit/(visit_id)/ @@ -468,7 +470,7 @@ ) @api_doc("/origin/intrinsic-metadata/") @format_docstring() -def api_origin_intrinsic_metadata(request, origin_url): +def api_origin_intrinsic_metadata(request: Request, origin_url: str): """ .. http:get:: /api/1/origin/(origin_url)/intrinsic-metadata diff --git a/swh/web/api/views/origin_save.py b/swh/web/api/views/origin_save.py --- a/swh/web/api/views/origin_save.py +++ b/swh/web/api/views/origin_save.py @@ -1,9 +1,12 @@ -# Copyright (C) 2018-2021 The Software Heritage developers +# Copyright (C) 2018-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os +from typing import Optional, cast + +from rest_framework.request import Request from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route @@ -19,7 +22,7 @@ ) -def _savable_visit_types(): +def _savable_visit_types() -> str: docstring = "" if os.environ.get("DJANGO_SETTINGS_MODULE") != "swh.web.settings.tests": visit_types = sorted(get_savable_visit_types()) @@ -39,7 +42,7 @@ ) @api_doc("/origin/save/") @format_docstring(visit_types=_savable_visit_types()) -def api_save_origin(request, visit_type, origin_url): +def api_save_origin(request: Request, visit_type: str, origin_url: str): """ .. http:get:: /api/1/origin/save/(visit_type)/url/(origin_url)/ .. http:post:: /api/1/origin/save/(visit_type)/url/(origin_url)/ @@ -112,13 +115,13 @@ request, permissions=[SWH_AMBASSADOR_PERMISSION, API_SAVE_ORIGIN_PERMISSION], ), - user_id=request.user.id, + user_id=cast(Optional[int], request.user.id), **data, ) del sor["id"] + return sor else: - sor = get_save_origin_requests(visit_type, origin_url) - for s in sor: - del s["id"] - - return sor + sors = get_save_origin_requests(visit_type, origin_url) + for sor in sors: + del sor["id"] + return sors diff --git a/swh/web/api/views/ping.py b/swh/web/api/views/ping.py --- a/swh/web/api/views/ping.py +++ b/swh/web/api/views/ping.py @@ -1,15 +1,17 @@ -# Copyright (C) 2020 The Software Heritage developers +# Copyright (C) 2020-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +from rest_framework.request import Request + from swh.web.api.apidoc import api_doc from swh.web.api.apiurls import api_route @api_route(r"/ping/", "api-1-ping") @api_doc("/ping/", noargs=True) -def ping(request): +def ping(request: Request): """ .. http:get:: /api/1/ping/ diff --git a/swh/web/api/views/raw.py b/swh/web/api/views/raw.py --- a/swh/web/api/views/raw.py +++ b/swh/web/api/views/raw.py @@ -5,6 +5,7 @@ from django.http import HttpResponse from rest_framework.exceptions import PermissionDenied +from rest_framework.request import Request from swh.model import model from swh.model.git_objects import ( @@ -33,7 +34,7 @@ ) @api_doc("/raw/") @format_docstring() -def api_raw_object(request, swhid): +def api_raw_object(request: Request, swhid: str): """ .. http:get:: /api/1/raw/(swhid)/ @@ -64,9 +65,9 @@ if not (request.user.is_staff or request.user.has_perm(API_RAW_OBJECT_PERMISSION)): raise PermissionDenied() - swhid = CoreSWHID.from_string(swhid) - object_id = swhid.object_id - object_type = swhid.object_type + parsed_swhid = CoreSWHID.from_string(swhid) + object_id = parsed_swhid.object_id + object_type = parsed_swhid.object_type def not_found(): return NotFoundExc(f"Object with id {swhid} not found.") @@ -85,34 +86,34 @@ result = content_git_object(cnt) elif object_type == ObjectType.DIRECTORY: - result = directory_get(archive.storage, object_id) - if result is None: + dir_ = directory_get(archive.storage, object_id) + if dir_ is None: raise not_found() - result = directory_git_object(result) + result = directory_git_object(dir_) elif object_type == ObjectType.REVISION: - result = archive.storage.revision_get([object_id], ignore_displayname=True)[0] - if result is None: + rev = archive.storage.revision_get([object_id], ignore_displayname=True)[0] + if rev is None: raise not_found() - result = revision_git_object(result) + result = revision_git_object(rev) elif object_type == ObjectType.RELEASE: - result = archive.storage.release_get([object_id], ignore_displayname=True)[0] - if result is None: + rel = archive.storage.release_get([object_id], ignore_displayname=True)[0] + if rel is None: raise not_found() - result = release_git_object(result) + result = release_git_object(rel) elif object_type == ObjectType.SNAPSHOT: - result = snapshot_get_all_branches(archive.storage, object_id) - if result is None: + snp = snapshot_get_all_branches(archive.storage, object_id) + if snp is None: raise not_found() - result = snapshot_git_object(result) + result = snapshot_git_object(snp) else: raise ValueError(f"Unexpected object type variant: {object_type}") response = HttpResponse(result, content_type="application/octet-stream") - filename = str(swhid).replace(":", "_") + "_raw" + filename = swhid.replace(":", "_") + "_raw" response["Content-disposition"] = f"attachment; filename={filename}" return response diff --git a/swh/web/api/views/release.py b/swh/web/api/views/release.py --- a/swh/web/api/views/release.py +++ b/swh/web/api/views/release.py @@ -1,8 +1,10 @@ -# Copyright (C) 2015-2019 The Software Heritage developers +# Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +from rest_framework.request import Request + from swh.web.api import utils from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route @@ -15,7 +17,7 @@ ) @api_doc("/release/") @format_docstring() -def api_release(request, sha1_git): +def api_release(request: Request, sha1_git: str): """ .. http:get:: /api/1/release/(sha1_git)/ diff --git a/swh/web/api/views/revision.py b/swh/web/api/views/revision.py --- a/swh/web/api/views/revision.py +++ b/swh/web/api/views/revision.py @@ -1,9 +1,12 @@ -# Copyright (C) 2015-2019 The Software Heritage developers +# Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +from typing import Optional + from django.http import HttpResponse +from rest_framework.request import Request from swh.web.api import utils from swh.web.api.apidoc import api_doc, format_docstring @@ -41,7 +44,7 @@ ) @api_doc("/revision/") @format_docstring(return_revision=DOC_RETURN_REVISION) -def api_revision(request, sha1_git): +def api_revision(request: Request, sha1_git: str): """ .. http:get:: /api/1/revision/(sha1_git)/ @@ -83,7 +86,7 @@ checksum_args=["sha1_git"], ) @api_doc("/revision/raw/", tags=["hidden"]) -def api_revision_raw_message(request, sha1_git): +def api_revision_raw_message(request: Request, sha1_git: str): """Return the raw data of the message of revision identified by sha1_git""" raw = archive.lookup_revision_message(sha1_git) response = HttpResponse(raw["message"], content_type="application/octet-stream") @@ -103,7 +106,9 @@ ) @api_doc("/revision/directory/") @format_docstring() -def api_revision_directory(request, sha1_git, dir_path=None, with_data=False): +def api_revision_directory( + request: Request, sha1_git: str, dir_path: Optional[str] = None +): """ .. http:get:: /api/1/revision/(sha1_git)/directory/[(path)/] @@ -136,7 +141,7 @@ :swh_web_api:`revision/f1b94134a4b879bc55c3dacdb496690c8ebdc03f/directory/` """ rev_id, result = archive.lookup_directory_through_revision( - {"sha1_git": sha1_git}, dir_path, with_data=with_data + {"sha1_git": sha1_git}, dir_path ) content = result["content"] @@ -159,7 +164,7 @@ ) @api_doc("/revision/log/") @format_docstring(return_revision_array=DOC_RETURN_REVISION_ARRAY) -def api_revision_log(request, sha1_git): +def api_revision_log(request: Request, sha1_git: str): """ .. http:get:: /api/1/revision/(sha1_git)/log/ diff --git a/swh/web/api/views/snapshot.py b/swh/web/api/views/snapshot.py --- a/swh/web/api/views/snapshot.py +++ b/swh/web/api/views/snapshot.py @@ -1,8 +1,10 @@ -# Copyright (C) 2018-2019 The Software Heritage developers +# Copyright (C) 2018-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +from rest_framework.request import Request + from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.api.utils import enrich_snapshot @@ -19,7 +21,7 @@ ) @api_doc("/snapshot/") @format_docstring() -def api_snapshot(request, snapshot_id): +def api_snapshot(request: Request, snapshot_id: str): """ .. http:get:: /api/1/snapshot/(snapshot_id)/ @@ -72,8 +74,8 @@ branches_from = request.GET.get("branches_from", "") branches_count = int(request.GET.get("branches_count", snapshot_content_max_size)) - target_types = request.GET.get("target_types", None) - target_types = target_types.split(",") if target_types else None + target_types_str = request.GET.get("target_types", None) + target_types = target_types_str.split(",") if target_types_str else None results = api_lookup( archive.lookup_snapshot, diff --git a/swh/web/api/views/stat.py b/swh/web/api/views/stat.py --- a/swh/web/api/views/stat.py +++ b/swh/web/api/views/stat.py @@ -1,8 +1,10 @@ -# Copyright (C) 2015-2019 The Software Heritage developers +# Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +from rest_framework.request import Request + from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.common import archive @@ -11,7 +13,7 @@ @api_route(r"/stat/counters/", "api-1-stat-counters") @api_doc("/stat/counters/", noargs=True) @format_docstring() -def api_stats(request): +def api_stats(request: Request): """ .. http:get:: /api/1/stat/counters/ diff --git a/swh/web/api/views/utils.py b/swh/web/api/views/utils.py --- a/swh/web/api/views/utils.py +++ b/swh/web/api/views/utils.py @@ -1,33 +1,32 @@ -# Copyright (C) 2015-2019 The Software Heritage developers +# Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from types import GeneratorType -from typing import Any, Callable, Dict, Mapping, Optional - -from typing_extensions import Protocol +from typing import Any, Callable, Dict, List, Optional, Tuple, Union from django.http import HttpRequest from rest_framework.decorators import api_view +from rest_framework.request import Request from rest_framework.response import Response from swh.web.api.apiurls import APIUrls, api_route from swh.web.common.exc import NotFoundExc +EnrichFunction = Callable[[Dict[str, str], Optional[HttpRequest]], Dict[str, str]] -class EnrichFunction(Protocol): - def __call__( - self, input: Mapping[str, str], request: Optional[HttpRequest] - ) -> Dict[str, str]: - ... +EnrichFunctionSearchResult = Callable[ + [Tuple[List[Dict[str, Any]], Optional[str]], Optional[HttpRequest]], + Tuple[List[Dict[str, Any]], Optional[str]], +] def api_lookup( lookup_fn: Callable[..., Any], *args: Any, notfound_msg: Optional[str] = "Object not found", - enrich_fn: Optional[EnrichFunction] = None, + enrich_fn: Optional[Union[EnrichFunction, EnrichFunctionSearchResult]] = None, request: Optional[HttpRequest] = None, **kwargs: Any, ): @@ -69,12 +68,12 @@ if res is None: raise NotFoundExc(notfound_msg) if isinstance(res, (list, GeneratorType)) or type(res) == map: - return [enrich_fn(x, request=request) for x in res] - return enrich_fn(res, request=request) + return [enrich_fn(x, request) for x in res] + return enrich_fn(res, request) @api_view(["GET", "HEAD"]) -def api_home(request): +def api_home(request: Request): return Response({}, template_name="api/api.html") diff --git a/swh/web/api/views/vault.py b/swh/web/api/views/vault.py --- a/swh/web/api/views/vault.py +++ b/swh/web/api/views/vault.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2021 The Software Heritage developers +# Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -7,6 +7,7 @@ from django.http import HttpResponse from django.shortcuts import redirect +from rest_framework.request import Request from swh.model.hashutil import hash_to_hex from swh.model.swhids import CoreSWHID, ObjectType @@ -74,7 +75,7 @@ ) @api_doc("/vault/flat/") @format_docstring() -def api_vault_cook_flat(request, swhid): +def api_vault_cook_flat(request: Request, swhid: str): """ .. http:get:: /api/1/vault/flat/(swhid)/ .. http:post:: /api/1/vault/flat/(swhid)/ @@ -115,21 +116,21 @@ request yet (in case of GET) or can not be found in the archive (in case of POST) """ - swhid = CoreSWHID.from_string(swhid) - if swhid.object_type == ObjectType.DIRECTORY: - res = _dispatch_cook_progress(request, "flat", swhid) + parsed_swhid = CoreSWHID.from_string(swhid) + if parsed_swhid.object_type == ObjectType.DIRECTORY: + res = _dispatch_cook_progress(request, "flat", parsed_swhid) res["fetch_url"] = reverse( "api-1-vault-fetch-flat", - url_args={"swhid": str(swhid)}, + url_args={"swhid": swhid}, request=request, ) return _vault_response(res, add_legacy_items=False) - elif swhid.object_type == ObjectType.CONTENT: + elif parsed_swhid.object_type == ObjectType.CONTENT: raise BadInputExc( "Content objects do not need to be cooked, " "use `/api/1/content/raw/` instead." ) - elif swhid.object_type == ObjectType.REVISION: + elif parsed_swhid.object_type == ObjectType.REVISION: # TODO: support revisions too? (the vault allows it) raise BadInputExc( "Only directories can be cooked as 'flat' bundles. " @@ -149,7 +150,7 @@ ) @api_doc("/vault/directory/", tags=["deprecated"]) @format_docstring() -def api_vault_cook_directory(request, dir_id): +def api_vault_cook_directory(request: Request, dir_id: str): """ .. http:get:: /api/1/vault/directory/(dir_id)/ @@ -174,7 +175,7 @@ "api-1-vault-fetch-flat", ) @api_doc("/vault/flat/raw/") -def api_vault_fetch_flat(request, swhid): +def api_vault_fetch_flat(request: Request, swhid: str): """ .. http:get:: /api/1/vault/flat/(swhid)/raw/ @@ -213,7 +214,7 @@ checksum_args=["dir_id"], ) @api_doc("/vault/directory/raw/", tags=["hidden", "deprecated"]) -def api_vault_fetch_directory(request, dir_id): +def api_vault_fetch_directory(request: Request, dir_id: str): """ .. http:get:: /api/1/vault/directory/(dir_id)/raw/ @@ -241,7 +242,7 @@ ) @api_doc("/vault/gitfast/") @format_docstring() -def api_vault_cook_gitfast(request, swhid): +def api_vault_cook_gitfast(request: Request, swhid: str): """ .. http:get:: /api/1/vault/gitfast/(swhid)/ .. http:post:: /api/1/vault/gitfast/(swhid)/ @@ -283,21 +284,21 @@ request yet (in case of GET) or can not be found in the archive (in case of POST) """ - swhid = CoreSWHID.from_string(swhid) - if swhid.object_type == ObjectType.REVISION: - res = _dispatch_cook_progress(request, "gitfast", swhid) + parsed_swhid = CoreSWHID.from_string(swhid) + if parsed_swhid.object_type == ObjectType.REVISION: + res = _dispatch_cook_progress(request, "gitfast", parsed_swhid) res["fetch_url"] = reverse( "api-1-vault-fetch-gitfast", - url_args={"swhid": str(swhid)}, + url_args={"swhid": swhid}, request=request, ) return _vault_response(res, add_legacy_items=False) - elif swhid.object_type == ObjectType.CONTENT: + elif parsed_swhid.object_type == ObjectType.CONTENT: raise BadInputExc( "Content objects do not need to be cooked, " "use `/api/1/content/raw/` instead." ) - elif swhid.object_type == ObjectType.DIRECTORY: + elif parsed_swhid.object_type == ObjectType.DIRECTORY: raise BadInputExc( "Only revisions can be cooked as 'gitfast' bundles. " "Use `/api/1/vault/flat/` to cook directories, as flat bundles." @@ -316,7 +317,7 @@ ) @api_doc("/vault/revision/gitfast/", tags=["deprecated"]) @format_docstring() -def api_vault_cook_revision_gitfast(request, rev_id): +def api_vault_cook_revision_gitfast(request: Request, rev_id: str): """ .. http:get:: /api/1/vault/revision/(rev_id)/gitfast/ @@ -341,7 +342,7 @@ "api-1-vault-fetch-gitfast", ) @api_doc("/vault/gitfast/raw/") -def api_vault_fetch_revision_gitfast(request, swhid): +def api_vault_fetch_revision_gitfast(request: Request, swhid: str): """ .. http:get:: /api/1/vault/gitfast/(swhid)/raw/ @@ -380,7 +381,7 @@ checksum_args=["rev_id"], ) @api_doc("/vault/revision_gitfast/raw/", tags=["hidden", "deprecated"]) -def _api_vault_revision_gitfast_raw(request, rev_id): +def _api_vault_revision_gitfast_raw(request: Request, rev_id: str): """ .. http:get:: /api/1/vault/revision/(rev_id)/gitfast/raw/ @@ -405,7 +406,7 @@ ) @api_doc("/vault/git-bare/") @format_docstring() -def api_vault_cook_git_bare(request, swhid): +def api_vault_cook_git_bare(request: Request, swhid: str): """ .. http:get:: /api/1/vault/git-bare/(swhid)/ .. http:post:: /api/1/vault/git-bare/(swhid)/ @@ -451,21 +452,21 @@ request yet (in case of GET) or can not be found in the archive (in case of POST) """ - swhid = CoreSWHID.from_string(swhid) - if swhid.object_type == ObjectType.REVISION: - res = _dispatch_cook_progress(request, "git_bare", swhid) + parsed_swhid = CoreSWHID.from_string(swhid) + if parsed_swhid.object_type == ObjectType.REVISION: + res = _dispatch_cook_progress(request, "git_bare", parsed_swhid) res["fetch_url"] = reverse( "api-1-vault-fetch-git-bare", - url_args={"swhid": str(swhid)}, + url_args={"swhid": swhid}, request=request, ) return _vault_response(res, add_legacy_items=False) - elif swhid.object_type == ObjectType.CONTENT: + elif parsed_swhid.object_type == ObjectType.CONTENT: raise BadInputExc( "Content objects do not need to be cooked, " "use `/api/1/content/raw/` instead." ) - elif swhid.object_type == ObjectType.DIRECTORY: + elif parsed_swhid.object_type == ObjectType.DIRECTORY: raise BadInputExc( "Only revisions can be cooked as 'git-bare' bundles. " "Use `/api/1/vault/flat/` to cook directories, as flat bundles." @@ -479,7 +480,7 @@ "api-1-vault-fetch-git-bare", ) @api_doc("/vault/git-bare/raw/") -def api_vault_fetch_revision_git_bare(request, swhid): +def api_vault_fetch_revision_git_bare(request: Request, swhid: str): """ .. http:get:: /api/1/vault/git-bare/(swhid)/raw/ diff --git a/swh/web/common/archive.py b/swh/web/common/archive.py --- a/swh/web/common/archive.py +++ b/swh/web/common/archive.py @@ -1406,7 +1406,7 @@ raise ValueError(f"Unexpected object type variant: {object_type}") -def lookup_missing_hashes(grouped_swhids: Dict[str, List[bytes]]) -> Set[str]: +def lookup_missing_hashes(grouped_swhids: Dict[ObjectType, List[bytes]]) -> Set[str]: """Lookup missing Software Heritage persistent identifier hash, using batch processing. diff --git a/swh/web/common/typing.py b/swh/web/common/typing.py --- a/swh/web/common/typing.py +++ b/swh/web/common/typing.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2021 The Software Heritage developers +# Copyright (C) 2020-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -229,7 +229,7 @@ PagedResult = CorePagedResult[TResult, str] -class SaveOriginRequestInfo(TypedDict): +class SaveOriginRequestInfo(TypedDict, total=False): id: int """Unique key""" save_request_date: str