diff --git a/swh/web/browse/snapshot_context.py b/swh/web/browse/snapshot_context.py --- a/swh/web/browse/snapshot_context.py +++ b/swh/web/browse/snapshot_context.py @@ -253,6 +253,8 @@ releases_info = service.lookup_release_multiple(release_to_branch.keys()) for release in releases_info: + if release is None: + continue branches_to_update = release_to_branch[release["id"]] for branch in branches_to_update: _add_release_info(branch, release) @@ -268,8 +270,8 @@ continue for branch in revision_to_branch[revision["id"]]: _add_branch_info(branch, revision) - for release in revision_to_release[revision["id"]]: - releases[release]["directory"] = revision["directory"] + for release_id in revision_to_release[revision["id"]]: + releases[release_id]["directory"] = revision["directory"] for branch_alias, branch_target in branch_aliases.items(): if branch_target in branches: diff --git a/swh/web/common/converters.py b/swh/web/common/converters.py --- a/swh/web/common/converters.py +++ b/swh/web/common/converters.py @@ -1,14 +1,16 @@ -# Copyright (C) 2015-2019 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import datetime import json -from typing import Dict, Any + +from typing import Any, Dict from swh.core.utils import decode_with_escape from swh.model import hashutil +from swh.model.model import Release from swh.storage.interface import PartialBranches from swh.web.common.typing import OriginInfo, OriginVisitInfo @@ -204,23 +206,14 @@ return from_swh(origin) -def from_release(release): +def from_release(release: Release) -> Dict[str, Any]: """Convert from a swh release to a json serializable release dictionary. Args: - release (dict): dictionary with keys: - - - id: identifier of the revision (sha1 in bytes) - - revision: identifier of the revision the release points to (sha1 - in bytes) - - comment: release's comment message (bytes) - name: release's name (string) - author: release's author identifier (swh's id) - synthetic: the synthetic property (boolean) + release: A release model object Returns: - dict: Release dictionary with the following keys: + release dictionary with the following keys - id: hexadecimal sha1 (string) - revision: hexadecimal sha1 (string) @@ -231,7 +224,7 @@ """ return from_swh( - release, + release.to_dict(), hashess={"id", "target"}, bytess={"message", "name", "fullname", "email"}, dates={"date"}, diff --git a/swh/web/common/service.py b/swh/web/common/service.py --- a/swh/web/common/service.py +++ b/swh/web/common/service.py @@ -446,7 +446,7 @@ return converters.from_directory_entry(queried_dir) -def lookup_release(release_sha1_git): +def lookup_release(release_sha1_git: str) -> Dict[str, Any]: """Return information about the release with sha1 release_sha1_git. Args: @@ -462,27 +462,31 @@ sha1_git_bin = _to_sha1_bin(release_sha1_git) release = _first_element(storage.release_get([sha1_git_bin])) if not release: - raise NotFoundExc("Release with sha1_git %s not found." % release_sha1_git) + raise NotFoundExc(f"Release with sha1_git {release_sha1_git} not found.") return converters.from_release(release) -def lookup_release_multiple(sha1_git_list): - """Return information about the revisions identified with +def lookup_release_multiple(sha1_git_list) -> Iterator[Optional[Dict[str, Any]]]: + """Return information about the releases identified with their sha1_git identifiers. Args: - sha1_git_list: A list of revision sha1_git identifiers + sha1_git_list: A list of release sha1_git identifiers Returns: - Release information as dict. + Iterator of Release metadata information as dict. Raises: ValueError if the identifier provided is not of sha1 nature. """ sha1_bin_list = [_to_sha1_bin(sha1_git) for sha1_git in sha1_git_list] - releases = storage.release_get(sha1_bin_list) or [] - return (converters.from_release(r) for r in releases) + releases = storage.release_get(sha1_bin_list) + for r in releases: + if r is not None: + yield converters.from_release(r) + else: + yield None def lookup_revision(rev_sha1_git): diff --git a/swh/web/tests/common/test_converters.py b/swh/web/tests/common/test_converters.py --- a/swh/web/tests/common/test_converters.py +++ b/swh/web/tests/common/test_converters.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2019 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -6,6 +6,13 @@ import datetime from swh.model import hashutil +from swh.model.model import ( + ObjectType, + Person, + Release, + TimestampWithTimezone, + Timestamp, +) from swh.web.common import converters @@ -200,26 +207,30 @@ def test_from_release(): - release_input = { - "id": hashutil.hash_to_bytes("aad23fa492a0c5fed0708a6703be875448c86884"), - "target": hashutil.hash_to_bytes("5e46d564378afc44b31bb89f99d5675195fbdf67"), - "target_type": "revision", - "date": { - "timestamp": datetime.datetime( - 2015, 1, 1, 22, 0, 0, tzinfo=datetime.timezone.utc - ).timestamp(), - "offset": 0, - "negative_utc": False, - }, - "author": { - "name": b"author name", - "fullname": b"Author Name author@email", - "email": b"author@email", - }, - "name": b"v0.0.1", - "message": b"some comment on release", - "synthetic": True, - } + """Convert release model object to a dict should be ok""" + ts = int( + datetime.datetime( + 2015, 1, 1, 22, 0, 0, tzinfo=datetime.timezone.utc + ).timestamp() + ) + release_input = Release( + id=hashutil.hash_to_bytes("aad23fa492a0c5fed0708a6703be875448c86884"), + target=hashutil.hash_to_bytes("5e46d564378afc44b31bb89f99d5675195fbdf67"), + target_type=ObjectType.REVISION, + date=TimestampWithTimezone( + timestamp=Timestamp(seconds=ts, microseconds=0), + offset=0, + negative_utc=False, + ), + author=Person( + name=b"author name", + fullname=b"Author Name author@email", + email=b"author@email", + ), + name=b"v0.0.1", + message=b"some comment on release", + synthetic=True, + ) expected_release = { "id": "aad23fa492a0c5fed0708a6703be875448c86884", @@ -242,46 +253,6 @@ assert actual_release == expected_release -def test_from_release_no_revision(): - release_input = { - "id": hashutil.hash_to_bytes("b2171ee2bdf119cd99a7ec7eff32fa8013ef9a4e"), - "target": None, - "date": { - "timestamp": datetime.datetime( - 2016, 3, 2, 10, 0, 0, tzinfo=datetime.timezone.utc - ).timestamp(), - "offset": 0, - "negative_utc": True, - }, - "name": b"v0.1.1", - "message": b"comment on release", - "synthetic": False, - "author": { - "name": b"bob", - "fullname": b"Bob bob@alice.net", - "email": b"bob@alice.net", - }, - } - - expected_release = { - "id": "b2171ee2bdf119cd99a7ec7eff32fa8013ef9a4e", - "target": None, - "date": "2016-03-02T10:00:00-00:00", - "name": "v0.1.1", - "message": "comment on release", - "synthetic": False, - "author": { - "name": "bob", - "fullname": "Bob bob@alice.net", - "email": "bob@alice.net", - }, - } - - actual_release = converters.from_release(release_input) - - assert actual_release == expected_release - - def test_from_revision(): revision_input = { "id": hashutil.hash_to_bytes("18d8be353ed3480476f032475e7c233eff7371d5"), diff --git a/swh/web/tests/common/test_service.py b/swh/web/tests/common/test_service.py --- a/swh/web/tests/common/test_service.py +++ b/swh/web/tests/common/test_service.py @@ -31,6 +31,7 @@ unknown_directory, release, unknown_release, + releases, revision, unknown_revision, revisions, @@ -256,6 +257,26 @@ assert e.match("Only sha1_git is supported.") +@given(releases()) +def test_lookup_release_multiple(archive_data, releases): + actual_releases = list(service.lookup_release_multiple(releases)) + + expected_releases = [] + for release_id in releases: + release_info = archive_data.release_get(release_id) + expected_releases.append(release_info) + + assert actual_releases == expected_releases + + +def test_lookup_release_multiple_none_found(): + unknown_releases_ = [random_sha1(), random_sha1(), random_sha1()] + + actual_releases = list(service.lookup_release_multiple(unknown_releases_)) + + assert actual_releases == [None] * len(unknown_releases_) + + @given(directory()) def test_lookup_directory_with_path_not_found(directory): path = "some/invalid/path/here" diff --git a/swh/web/tests/conftest.py b/swh/web/tests/conftest.py --- a/swh/web/tests/conftest.py +++ b/swh/web/tests/conftest.py @@ -205,10 +205,10 @@ ) return list(dir_content) - def release_get(self, rel_id): + def release_get(self, rel_id: str) -> Optional[Dict[str, Any]]: rel_id_bytes = hash_to_bytes(rel_id) - rel_data = next(self.storage.release_get([rel_id_bytes])) - return converters.from_release(rel_data) + rel_data = self.storage.release_get([rel_id_bytes])[0] + return converters.from_release(rel_data) if rel_data else None def revision_get(self, rev_id): rev_id_bytes = hash_to_bytes(rev_id) diff --git a/swh/web/tests/data.py b/swh/web/tests/data.py --- a/swh/web/tests/data.py +++ b/swh/web/tests/data.py @@ -227,8 +227,8 @@ if target_type == "revision": revisions.add(branch_data.target) elif target_type == "release": - release = next(storage.release_get([branch_data.target])) - revisions.add(release["target"]) + release = storage.release_get([branch_data.target])[0] + revisions.add(release.target) releases.add(hash_to_hex(branch_data.target)) for rev_log in storage.revision_shortlog(set(revisions)): diff --git a/swh/web/tests/strategies.py b/swh/web/tests/strategies.py --- a/swh/web/tests/strategies.py +++ b/swh/web/tests/strategies.py @@ -336,13 +336,21 @@ return _known_swh_object("releases") +def releases(min_size=2, max_size=8): + """ + Hypothesis strategy returning random releases ingested + into the test archive. + """ + return lists(release(), min_size=min_size, max_size=max_size) + + def unknown_release(): """ Hypothesis strategy returning a random revision not ingested into the test archive. """ return sha1().filter( - lambda s: next(get_tests_data()["storage"].release_get([s])) is None + lambda s: get_tests_data()["storage"].release_get([s])[0] is None )