diff --git a/swh/web/tests/api/views/test_content.py b/swh/web/tests/api/views/test_content.py --- a/swh/web/tests/api/views/test_content.py +++ b/swh/web/tests/api/views/test_content.py @@ -230,7 +230,7 @@ assert rv.status_code == 200, rv.data assert rv["Content-Type"] == "application/json" - expected_data = archive_data.content_get_metadata(content["sha1"]) + expected_data = archive_data.content_get(content["sha1"]) for key, view_name in ( ("data_url", "api-1-content-raw"), ("license_url", "api-1-content-license"), diff --git a/swh/web/tests/browse/views/test_content.py b/swh/web/tests/browse/views/test_content.py --- a/swh/web/tests/browse/views/test_content.py +++ b/swh/web/tests/browse/views/test_content.py @@ -326,8 +326,7 @@ @given(content()) def test_content_bytes_missing(client, archive_data, mocker, content): mock_service = mocker.patch("swh.web.browse.utils.service") - content_data = archive_data.content_get_metadata(content["sha1"]) - content_data["data"] = None + content_data = archive_data.content_get(content["sha1"]) mock_service.lookup_content.return_value = content_data mock_service.lookup_content_filetype.side_effect = Exception() diff --git a/swh/web/tests/common/test_service.py b/swh/web/tests/common/test_service.py --- a/swh/web/tests/common/test_service.py +++ b/swh/web/tests/common/test_service.py @@ -84,7 +84,7 @@ def test_lookup_hash_exist(archive_data, content): actual_lookup = service.lookup_hash("sha1:%s" % content["sha1"]) - content_metadata = archive_data.content_get_metadata(content["sha1"]) + content_metadata = archive_data.content_get(content["sha1"]) assert {"found": content_metadata, "algo": "sha1"} == actual_lookup @@ -619,18 +619,18 @@ @given(content()) def test_lookup_content_with_sha1(archive_data, content): - actual_content = service.lookup_content("sha1:%s" % content["sha1"]) + actual_content = service.lookup_content(f"sha1:{content['sha1']}") - expected_content = archive_data.content_get_metadata(content["sha1"]) + expected_content = archive_data.content_get(content["sha1"]) assert actual_content == expected_content @given(content()) def test_lookup_content_with_sha256(archive_data, content): - actual_content = service.lookup_content("sha256:%s" % content["sha256"]) + actual_content = service.lookup_content(f"sha256:{content['sha256']}") - expected_content = archive_data.content_get_metadata(content["sha1"]) + expected_content = archive_data.content_get(content["sha1"]) assert actual_content == expected_content diff --git a/swh/web/tests/conftest.py b/swh/web/tests/conftest.py --- a/swh/web/tests/conftest.py +++ b/swh/web/tests/conftest.py @@ -176,13 +176,16 @@ cnt = self.storage.content_find(cnt_ids_bytes) return converters.from_content(cnt[0].to_dict()) if cnt else cnt - def content_get_metadata(self, cnt_id): + def content_get(self, cnt_id: str) -> Dict[str, Any]: cnt_id_bytes = hash_to_bytes(cnt_id) - metadata = self.storage.content_get_metadata([cnt_id_bytes]) - contents = metadata[cnt_id_bytes] - content = None if not contents else contents[0] + content = self.storage.content_get([cnt_id_bytes])[0] + if content: + content_d = content.to_dict() + content_d.pop("ctime", None) + else: + content_d = None return converters.from_swh( - content, hashess={"sha1", "sha1_git", "sha256", "blake2s256"} + content_d, hashess={"sha1", "sha1_git", "sha256", "blake2s256"} ) def content_get_data(self, cnt_id: str) -> Optional[Dict[str, Any]]: diff --git a/swh/web/tests/data.py b/swh/web/tests/data.py --- a/swh/web/tests/data.py +++ b/swh/web/tests/data.py @@ -3,24 +3,25 @@ # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -from datetime import timedelta import os import random import time from copy import deepcopy +from datetime import timedelta +from typing import Dict, List, Optional, Set from swh.indexer.fossology_license import FossologyLicenseIndexer from swh.indexer.mimetype import MimetypeIndexer from swh.indexer.ctags import CtagsIndexer from swh.indexer.storage import get_indexer_storage -from swh.model.model import Content, OriginVisitStatus from swh.model.hashutil import hash_to_hex, hash_to_bytes, DEFAULT_ALGORITHMS -from swh.model.model import Directory, Origin, OriginVisit +from swh.model.model import Content, Directory, Origin, OriginVisit, OriginVisitStatus from swh.loader.git.from_disk import GitLoaderFromArchive from swh.search import get_search from swh.storage.algos.dir_iterators import dir_iterator from swh.storage.algos.snapshot import snapshot_get_latest +from swh.storage.interface import Sha1 from swh.storage.utils import now from swh.web import config from swh.web.browse.utils import ( @@ -209,7 +210,7 @@ ) storage.origin_visit_status_add([visit_status]) - contents = set() + sha1s: Set[Sha1] = set() directories = set() revisions = set() releases = set() @@ -239,28 +240,31 @@ directories.add(hash_to_hex(dir_id)) for entry in dir_iterator(storage, dir_id): if entry["type"] == "file": - contents.add(entry["sha1"]) + sha1s.add(entry["sha1"]) content_path[entry["sha1"]] = "/".join( [hash_to_hex(dir_id), entry["path"].decode("utf-8")] ) elif entry["type"] == "dir": directories.add(hash_to_hex(entry["target"])) - _add_extra_contents(storage, contents) + _add_extra_contents(storage, sha1s) # Get all checksums for each content - result = storage.content_get_metadata(contents) - contents = [] - for sha1, contents_metadata in result.items(): - sha1 = contents_metadata[0]["sha1"] + result: List[Optional[Content]] = storage.content_get(list(sha1s)) + + contents: List[Dict] = [] + for content in result: + assert content is not None + sha1 = hash_to_hex(content.sha1) content_metadata = { - algo: hash_to_hex(contents_metadata[0][algo]) for algo in DEFAULT_ALGORITHMS + algo: hash_to_hex(getattr(content, algo)) for algo in DEFAULT_ALGORITHMS } path = "" - if sha1 in content_path: - path = content_path[sha1] - cnt_data = storage.content_get_data(sha1) + if content.sha1 in content_path: + path = content_path[content.sha1] + + cnt_data = storage.content_get_data(content.sha1) assert cnt_data is not None mimetype, encoding = get_mimetype_and_encoding_for_content(cnt_data) _, _, cnt_data = _re_encode_content(mimetype, encoding, cnt_data) @@ -275,7 +279,7 @@ "data": content_display_data["content_data"], } ) - _contents[hash_to_hex(sha1)] = content_metadata + _contents[sha1] = content_metadata contents.append(content_metadata) # Create indexer storage instance that will be shared by indexers