Changeset View
Changeset View
Standalone View
Standalone View
swh/web/common/service.py
| # Copyright (C) 2015-2020 The Software Heritage developers | # Copyright (C) 2015-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU Affero General Public License version 3, or any later version | # License: GNU Affero General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import itertools | import itertools | ||||
| import os | import os | ||||
| import re | import re | ||||
| from collections import defaultdict | from collections import defaultdict | ||||
| from typing import Any, Dict, List, Set, Iterable, Iterator, Optional, Tuple | from typing import Any, Dict, List, Set, Iterable, Iterator, Optional, Union, Tuple | ||||
| from swh.model import hashutil | from swh.model import hashutil | ||||
| from swh.model.identifiers import CONTENT, DIRECTORY, RELEASE, REVISION, SNAPSHOT | from swh.model.identifiers import CONTENT, DIRECTORY, RELEASE, REVISION, SNAPSHOT | ||||
| from swh.model.model import OriginVisit | from swh.model.model import OriginVisit, Revision | ||||
| from swh.storage.algos import diff, revisions_walker | from swh.storage.algos import diff, revisions_walker | ||||
| from swh.storage.algos.origin import origin_get_latest_visit_status | from swh.storage.algos.origin import origin_get_latest_visit_status | ||||
| from swh.storage.algos.snapshot import snapshot_get_latest | from swh.storage.algos.snapshot import snapshot_get_latest | ||||
| from swh.vault.exc import NotFoundExc as VaultNotFoundExc | from swh.vault.exc import NotFoundExc as VaultNotFoundExc | ||||
| from swh.web import config | from swh.web import config | ||||
| from swh.web.common import converters | from swh.web.common import converters | ||||
| from swh.web.common import query | from swh.web.common import query | ||||
| from swh.web.common.exc import BadInputExc, NotFoundExc | from swh.web.common.exc import BadInputExc, NotFoundExc | ||||
| ▲ Show 20 Lines • Show All 460 Lines • ▼ Show 20 Lines | def lookup_release_multiple(sha1_git_list) -> Iterator[Optional[Dict[str, Any]]]: | ||||
| releases = storage.release_get(sha1_bin_list) | releases = storage.release_get(sha1_bin_list) | ||||
| for r in releases: | for r in releases: | ||||
| if r is not None: | if r is not None: | ||||
| yield converters.from_release(r) | yield converters.from_release(r) | ||||
| else: | else: | ||||
| yield None | yield None | ||||
| def lookup_revision(rev_sha1_git): | def lookup_revision(rev_sha1_git) -> Dict[str, Any]: | ||||
| """Return information about the revision with sha1 revision_sha1_git. | """Return information about the revision with sha1 revision_sha1_git. | ||||
| Args: | Args: | ||||
| revision_sha1_git: The revision's sha1 as hexadecimal | revision_sha1_git: The revision's sha1 as hexadecimal | ||||
| Returns: | Returns: | ||||
| Revision information as dict. | Revision information as dict. | ||||
| Raises: | Raises: | ||||
| ValueError if the identifier provided is not of sha1 nature. | ValueError if the identifier provided is not of sha1 nature. | ||||
| NotFoundExc if there is no revision with the provided sha1_git. | NotFoundExc if there is no revision with the provided sha1_git. | ||||
| """ | """ | ||||
| sha1_git_bin = _to_sha1_bin(rev_sha1_git) | sha1_git_bin = _to_sha1_bin(rev_sha1_git) | ||||
| revision = _first_element(storage.revision_get([sha1_git_bin])) | revision = storage.revision_get([sha1_git_bin])[0] | ||||
| if not revision: | if not revision: | ||||
| raise NotFoundExc("Revision with sha1_git %s not found." % rev_sha1_git) | raise NotFoundExc(f"Revision with sha1_git {rev_sha1_git} not found.") | ||||
| return converters.from_revision(revision) | return converters.from_revision(revision) | ||||
| def lookup_revision_multiple(sha1_git_list): | def lookup_revision_multiple(sha1_git_list) -> Iterator[Optional[Dict[str, Any]]]: | ||||
| """Return information about the revisions identified with | """Return information about the revisions identified with | ||||
| their sha1_git identifiers. | their sha1_git identifiers. | ||||
| Args: | Args: | ||||
| sha1_git_list: A list of revision sha1_git identifiers | sha1_git_list: A list of revision sha1_git identifiers | ||||
| Returns: | Yields: | ||||
| Iterator of revisions information as dict. | revision information as dict if the revision exists, None otherwise. | ||||
| Raises: | Raises: | ||||
| ValueError if the identifier provided is not of sha1 nature. | ValueError if the identifier provided is not of sha1 nature. | ||||
| """ | """ | ||||
| sha1_bin_list = [_to_sha1_bin(sha1_git) for sha1_git in sha1_git_list] | sha1_bin_list = [_to_sha1_bin(sha1_git) for sha1_git in sha1_git_list] | ||||
| revisions = storage.revision_get(sha1_bin_list) or [] | revisions = storage.revision_get(sha1_bin_list) | ||||
| return (converters.from_revision(r) for r in revisions) | for revision in revisions: | ||||
| if revision is not None: | |||||
| yield converters.from_revision(revision) | |||||
| else: | |||||
| yield None | |||||
| def lookup_revision_message(rev_sha1_git): | def lookup_revision_message(rev_sha1_git) -> Dict[str, bytes]: | ||||
| """Return the raw message of the revision with sha1 revision_sha1_git. | """Return the raw message of the revision with sha1 revision_sha1_git. | ||||
| Args: | Args: | ||||
| revision_sha1_git: The revision's sha1 as hexadecimal | revision_sha1_git: The revision's sha1 as hexadecimal | ||||
| Returns: | Returns: | ||||
| Decoded revision message as dict {'message': <the_message>} | Decoded revision message as dict {'message': <the_message>} | ||||
| Raises: | Raises: | ||||
| ValueError if the identifier provided is not of sha1 nature. | ValueError if the identifier provided is not of sha1 nature. | ||||
| NotFoundExc if the revision is not found, or if it has no message | NotFoundExc if the revision is not found, or if it has no message | ||||
| """ | """ | ||||
| sha1_git_bin = _to_sha1_bin(rev_sha1_git) | sha1_git_bin = _to_sha1_bin(rev_sha1_git) | ||||
| revision = storage.revision_get([sha1_git_bin])[0] | |||||
| revision = _first_element(storage.revision_get([sha1_git_bin])) | |||||
| if not revision: | if not revision: | ||||
| raise NotFoundExc("Revision with sha1_git %s not found." % rev_sha1_git) | raise NotFoundExc(f"Revision with sha1_git {rev_sha1_git} not found.") | ||||
| if "message" not in revision: | if not revision.message: | ||||
| raise NotFoundExc("No message for revision with sha1_git %s." % rev_sha1_git) | raise NotFoundExc(f"No message for revision with sha1_git {rev_sha1_git}.") | ||||
| res = {"message": revision["message"]} | return {"message": revision.message} | ||||
| return res | |||||
| def _lookup_revision_id_by(origin, branch_name, timestamp): | def _lookup_revision_id_by(origin, branch_name, timestamp): | ||||
| def _get_snapshot_branch(snapshot, branch_name): | def _get_snapshot_branch(snapshot, branch_name): | ||||
| snapshot = lookup_snapshot( | snapshot = lookup_snapshot( | ||||
| visit["snapshot"], branches_from=branch_name, branches_count=10 | visit["snapshot"], branches_from=branch_name, branches_count=10 | ||||
| ) | ) | ||||
| branch = None | branch = None | ||||
| ▲ Show 20 Lines • Show All 121 Lines • ▼ Show 20 Lines | Raises: | ||||
| - NotFoundExc if either revision is not found or if sha1_git is not an | - NotFoundExc if either revision is not found or if sha1_git is not an | ||||
| ancestor of sha1_git_root. | ancestor of sha1_git_root. | ||||
| """ | """ | ||||
| rev_root_id = _lookup_revision_id_by(origin, branch_name, timestamp) | rev_root_id = _lookup_revision_id_by(origin, branch_name, timestamp) | ||||
| rev_root_id_bin = hashutil.hash_to_bytes(rev_root_id) | rev_root_id_bin = hashutil.hash_to_bytes(rev_root_id) | ||||
| rev_root = _first_element(storage.revision_get([rev_root_id_bin])) | rev_root = storage.revision_get([rev_root_id_bin])[0] | ||||
| return ( | return ( | ||||
| converters.from_revision(rev_root), | converters.from_revision(rev_root) if rev_root else None, | ||||
| lookup_revision_with_context(rev_root, sha1_git, limit), | lookup_revision_with_context(rev_root, sha1_git, limit), | ||||
| ) | ) | ||||
| def lookup_revision_with_context(sha1_git_root, sha1_git, limit=100): | def lookup_revision_with_context( | ||||
| sha1_git_root: Union[str, Dict[str, Any], Revision], sha1_git: str, limit: int = 100 | |||||
| ) -> Dict[str, Any]: | |||||
| """Return information about revision sha1_git, limited to the | """Return information about revision sha1_git, limited to the | ||||
| sub-graph of all transitive parents of sha1_git_root. | sub-graph of all transitive parents of sha1_git_root. | ||||
| In other words, sha1_git is an ancestor of sha1_git_root. | In other words, sha1_git is an ancestor of sha1_git_root. | ||||
| Args: | Args: | ||||
| sha1_git_root: latest revision. The type is either a sha1 (as an hex | sha1_git_root: latest revision. The type is either a sha1 (as an hex | ||||
| string) or a non converted dict. | string) or a non converted dict. | ||||
| sha1_git: one of sha1_git_root's ancestors | sha1_git: one of sha1_git_root's ancestors | ||||
| limit: limit the lookup to 100 revisions back | limit: limit the lookup to 100 revisions back | ||||
| Returns: | Returns: | ||||
| Information on sha1_git if it is an ancestor of sha1_git_root | Information on sha1_git if it is an ancestor of sha1_git_root | ||||
| including children leading to sha1_git_root | including children leading to sha1_git_root | ||||
| Raises: | Raises: | ||||
| BadInputExc in case of unknown algo_hash or bad hash | BadInputExc in case of unknown algo_hash or bad hash | ||||
| NotFoundExc if either revision is not found or if sha1_git is not an | NotFoundExc if either revision is not found or if sha1_git is not an | ||||
| ancestor of sha1_git_root | ancestor of sha1_git_root | ||||
| """ | """ | ||||
| sha1_git_bin = _to_sha1_bin(sha1_git) | sha1_git_bin = _to_sha1_bin(sha1_git) | ||||
| revision = _first_element(storage.revision_get([sha1_git_bin])) | revision = storage.revision_get([sha1_git_bin])[0] | ||||
| if not revision: | if not revision: | ||||
| raise NotFoundExc("Revision %s not found" % sha1_git) | raise NotFoundExc(f"Revision {sha1_git} not found") | ||||
| if isinstance(sha1_git_root, str): | if isinstance(sha1_git_root, str): | ||||
| sha1_git_root_bin = _to_sha1_bin(sha1_git_root) | sha1_git_root_bin = _to_sha1_bin(sha1_git_root) | ||||
| revision_root = _first_element(storage.revision_get([sha1_git_root_bin])) | revision_root = storage.revision_get([sha1_git_root_bin])[0] | ||||
| if not revision_root: | if not revision_root: | ||||
| raise NotFoundExc("Revision root %s not found" % sha1_git_root) | raise NotFoundExc(f"Revision root {sha1_git_root} not found") | ||||
| elif isinstance(sha1_git_root, Revision): | |||||
| sha1_git_root_bin = sha1_git_root.id | |||||
| else: | else: | ||||
| sha1_git_root_bin = sha1_git_root["id"] | sha1_git_root_bin = sha1_git_root["id"] | ||||
| revision_log = storage.revision_log([sha1_git_root_bin], limit) | revision_log = storage.revision_log([sha1_git_root_bin], limit) | ||||
| parents = {} | parents: Dict[str, List[str]] = {} | ||||
| children = defaultdict(list) | children = defaultdict(list) | ||||
| for rev in revision_log: | for rev in revision_log: | ||||
| rev_id = rev["id"] | rev_id = rev["id"] | ||||
| parents[rev_id] = [] | parents[rev_id] = [] | ||||
| for parent_id in rev["parents"]: | for parent_id in rev["parents"]: | ||||
| parents[rev_id].append(parent_id) | parents[rev_id].append(parent_id) | ||||
| children[parent_id].append(rev_id) | children[parent_id].append(rev_id) | ||||
| if revision["id"] not in parents: | if revision.id not in parents: | ||||
| raise NotFoundExc( | raise NotFoundExc(f"Revision {sha1_git} is not an ancestor of {sha1_git_root}") | ||||
| "Revision %s is not an ancestor of %s" % (sha1_git, sha1_git_root) | |||||
| ) | |||||
| revision["children"] = children[revision["id"]] | revision_d = revision.to_dict() | ||||
| revision_d["children"] = children[revision.id] | |||||
| return converters.from_revision(revision) | return converters.from_revision(revision_d) | ||||
| def lookup_directory_with_revision(sha1_git, dir_path=None, with_data=False): | def lookup_directory_with_revision(sha1_git, dir_path=None, with_data=False): | ||||
| """Return information on directory pointed by revision with sha1_git. | """Return information on directory pointed by revision with sha1_git. | ||||
| If dir_path is not provided, display top level directory. | If dir_path is not provided, display top level directory. | ||||
| Otherwise, display the directory pointed by dir_path (if it exists). | Otherwise, display the directory pointed by dir_path (if it exists). | ||||
| Args: | Args: | ||||
| Show All 9 Lines | Raises: | ||||
| BadInputExc in case of unknown algo_hash or bad hash. | BadInputExc in case of unknown algo_hash or bad hash. | ||||
| NotFoundExc either if the revision is not found or the path referenced | NotFoundExc either if the revision is not found or the path referenced | ||||
| does not exist. | does not exist. | ||||
| NotImplementedError in case of dir_path exists but do not reference a | NotImplementedError in case of dir_path exists but do not reference a | ||||
| type 'dir' or 'file'. | type 'dir' or 'file'. | ||||
| """ | """ | ||||
| sha1_git_bin = _to_sha1_bin(sha1_git) | sha1_git_bin = _to_sha1_bin(sha1_git) | ||||
| revision = _first_element(storage.revision_get([sha1_git_bin])) | revision = storage.revision_get([sha1_git_bin])[0] | ||||
| if not revision: | if not revision: | ||||
| raise NotFoundExc("Revision %s not found" % sha1_git) | raise NotFoundExc(f"Revision {sha1_git} not found") | ||||
| dir_sha1_git_bin = revision["directory"] | dir_sha1_git_bin = revision.directory | ||||
| if dir_path: | if dir_path: | ||||
| paths = dir_path.strip(os.path.sep).split(os.path.sep) | paths = dir_path.strip(os.path.sep).split(os.path.sep) | ||||
| entity = storage.directory_entry_get_by_path( | entity = storage.directory_entry_get_by_path( | ||||
| dir_sha1_git_bin, list(map(lambda p: p.encode("utf-8"), paths)) | dir_sha1_git_bin, list(map(lambda p: p.encode("utf-8"), paths)) | ||||
| ) | ) | ||||
| if not entity: | if not entity: | ||||
| raise NotFoundExc( | raise NotFoundExc( | ||||
| "Directory or File '%s' pointed to by revision %s not found" | "Directory or File '%s' pointed to by revision %s not found" | ||||
| Show All 20 Lines | elif entity["type"] == "file": # content | ||||
| content_d["data"] = data | content_d["data"] = data | ||||
| return { | return { | ||||
| "type": "file", | "type": "file", | ||||
| "path": "." if not dir_path else dir_path, | "path": "." if not dir_path else dir_path, | ||||
| "revision": sha1_git, | "revision": sha1_git, | ||||
| "content": converters.from_content(content_d), | "content": converters.from_content(content_d), | ||||
| } | } | ||||
| elif entity["type"] == "rev": # revision | elif entity["type"] == "rev": # revision | ||||
| revision = next(storage.revision_get([entity["target"]])) | revision = storage.revision_get([entity["target"]])[0] | ||||
| return { | return { | ||||
| "type": "rev", | "type": "rev", | ||||
| "path": "." if not dir_path else dir_path, | "path": "." if not dir_path else dir_path, | ||||
| "revision": sha1_git, | "revision": sha1_git, | ||||
| "content": converters.from_revision(revision), | "content": converters.from_revision(revision) if revision else None, | ||||
| } | } | ||||
| else: | else: | ||||
| raise NotImplementedError("Entity of type %s not implemented." % entity["type"]) | raise NotImplementedError("Entity of type %s not implemented." % entity["type"]) | ||||
| def lookup_content(q: str) -> Dict[str, Any]: | def lookup_content(q: str) -> Dict[str, Any]: | ||||
| """Lookup the content designed by q. | """Lookup the content designed by q. | ||||
| ▲ Show 20 Lines • Show All 508 Lines • Show Last 20 Lines | |||||