diff --git a/swh/fuse/fs/artifact.py b/swh/fuse/fs/artifact.py --- a/swh/fuse/fs/artifact.py +++ b/swh/fuse/fs/artifact.py @@ -10,7 +10,7 @@ from swh.fuse.fs.entry import EntryMode, FuseEntry from swh.fuse.fs.symlink import SymlinkEntry from swh.model.from_disk import DentryPerms -from swh.model.identifiers import CONTENT, DIRECTORY, REVISION, SWHID +from swh.model.identifiers import CONTENT, DIRECTORY, RELEASE, REVISION, SWHID @dataclass @@ -176,4 +176,78 @@ ) -OBJTYPE_GETTERS = {CONTENT: Content, DIRECTORY: Directory, REVISION: Revision} +class Release(ArtifactEntry): + """ Software Heritage release artifact. + + Release nodes are represented on the file-system as directories with the + following entries: + + - `target`: target node, as a symlink to `archive/` + - `target_type`: regular file containing the type of the target SWHID + - `root`: present if and only if the release points to something that + (transitively) resolves to a directory. When present it is a symlink + pointing into `archive/` to the SWHID of the given directory + - `meta.json`: metadata for the current node, as a symlink pointing to the + relevant `meta/.json` file """ + + async def find_root_directory(self, swhid: SWHID) -> SWHID: + if swhid.object_type == RELEASE: + metadata = await self.fuse.get_metadata(swhid) + return await self.find_root_directory(metadata["target"]) + elif swhid.object_type == REVISION: + metadata = await self.fuse.get_metadata(swhid) + return metadata["directory"] + elif swhid.object_type == DIRECTORY: + return swhid + else: + return None + + async def __aiter__(self) -> AsyncIterator[FuseEntry]: + metadata = await self.fuse.get_metadata(self.swhid) + root_path = self.get_relative_root_path() + + yield self.create_child( + SymlinkEntry, + name="meta.json", + target=Path(root_path, f"meta/{self.swhid}.json"), + ) + + target = metadata["target"] + yield self.create_child( + SymlinkEntry, name="target", target=Path(root_path, f"archive/{target}") + ) + yield self.create_child( + ReleaseType, + name="target_type", + mode=int(EntryMode.RDONLY_FILE), + target_type=target.object_type, + ) + + target_dir = await self.find_root_directory(target) + if target_dir is not None: + yield self.create_child( + SymlinkEntry, + name="root", + target=Path(root_path, f"archive/{target_dir}"), + ) + + +@dataclass +class ReleaseType(FuseEntry): + """ Release type virtual file """ + + target_type: str + + async def get_content(self) -> bytes: + return str.encode(self.target_type + "\n") + + async def size(self) -> int: + return len(await self.get_content()) + + +OBJTYPE_GETTERS = { + CONTENT: Content, + DIRECTORY: Directory, + REVISION: Revision, + RELEASE: Release, +} diff --git a/swh/fuse/tests/common.py b/swh/fuse/tests/common.py --- a/swh/fuse/tests/common.py +++ b/swh/fuse/tests/common.py @@ -3,7 +3,7 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from typing import Any +from typing import Any, List from swh.fuse.tests.data.api_data import MOCK_ARCHIVE, SWHID2URL @@ -13,3 +13,8 @@ if raw: url += "raw/" return MOCK_ARCHIVE[url] + + +def get_dir_name_entries(swhid: str) -> List[str]: + dir_meta = get_data_from_archive(swhid) + return [x["name"] for x in dir_meta] diff --git a/swh/fuse/tests/data/api_data.py b/swh/fuse/tests/data/api_data.py --- a/swh/fuse/tests/data/api_data.py +++ b/swh/fuse/tests/data/api_data.py @@ -31,6 +31,7 @@ "swh:1:rev:2d39e2894830331fb02b77980a6190e972ad3d68": "revision/2d39e2894830331fb02b77980a6190e972ad3d68/", "swh:1:rev:92baf7293dd2d418d2ac4b141b0faa822075d9f7": "revision/92baf7293dd2d418d2ac4b141b0faa822075d9f7/", "swh:1:rev:cf6447aff01e4bcb1fdbc89d6f754451a157589e": "revision/cf6447aff01e4bcb1fdbc89d6f754451a157589e/", + "swh:1:rel:874f7cbe352033cac5a8bc889847da2fe1d13e9f": "release/874f7cbe352033cac5a8bc889847da2fe1d13e9f/", } MOCK_ARCHIVE = { @@ -1737,4 +1738,19 @@ "history_url": "https://archive.softwareheritage.org/api/1/revision/cf6447aff01e4bcb1fdbc89d6f754451a157589e/log/", "directory_url": "https://archive.softwareheritage.org/api/1/directory/59263209d6c932eefc716826aa0d0df60e540cbe/", }, + "release/874f7cbe352033cac5a8bc889847da2fe1d13e9f/": { + "name": "1.42.0", + "message": "1.42.0 release\n", + "target": "b8cedc00407a4c56a3bda1ed605c6fc166655447", + "target_type": "revision", + "synthetic": False, + "author": { + "fullname": "Pietro Albini ", + "name": "Pietro Albini", + "email": "pietro@pietroalbini.org", + }, + "date": "2020-03-12T15:05:24+01:00", + "id": "874f7cbe352033cac5a8bc889847da2fe1d13e9f", + "target_url": "https://archive.softwareheritage.org/api/1/revision/b8cedc00407a4c56a3bda1ed605c6fc166655447/", + }, } diff --git a/swh/fuse/tests/data/config.py b/swh/fuse/tests/data/config.py --- a/swh/fuse/tests/data/config.py +++ b/swh/fuse/tests/data/config.py @@ -22,8 +22,16 @@ "swh:1:rev:92baf7293dd2d418d2ac4b141b0faa822075d9f7", ] # Release +ROOT_REL = "swh:1:rel:874f7cbe352033cac5a8bc889847da2fe1d13e9f" # TODO # Snapshot # TODO -ALL_ENTRIES = [REGULAR_FILE, ROOT_DIR, DIR_WITH_SUBMODULES, ROOT_REV, *SUBMODULES] +ALL_ENTRIES = [ + REGULAR_FILE, + ROOT_DIR, + DIR_WITH_SUBMODULES, + ROOT_REV, + *SUBMODULES, + ROOT_REL, +] diff --git a/swh/fuse/tests/data/gen-api-data.py b/swh/fuse/tests/data/gen-api-data.py --- a/swh/fuse/tests/data/gen-api-data.py +++ b/swh/fuse/tests/data/gen-api-data.py @@ -12,7 +12,14 @@ import requests from swh.fuse.tests.data.config import ALL_ENTRIES -from swh.model.identifiers import CONTENT, DIRECTORY, REVISION, SWHID, parse_swhid +from swh.model.identifiers import ( + CONTENT, + DIRECTORY, + RELEASE, + REVISION, + SWHID, + parse_swhid, +) API_URL_real = "https://archive.softwareheritage.org/api/1" API_URL_test = "https://invalid-test-only.archive.softwareheritage.org/api/1" @@ -28,12 +35,29 @@ CONTENT: "content/sha1_git:", DIRECTORY: "directory/", REVISION: "revision/", + RELEASE: "release/", } return f"{prefix[swhid.object_type]}{swhid.object_id}/" -def generate_archive_data(swhid: SWHID, raw: bool = False) -> None: +def get_short_type(object_type: str) -> str: + short_type = { + CONTENT: "cnt", + DIRECTORY: "dir", + REVISION: "rev", + RELEASE: "rel", + } + return short_type[object_type] + + +def generate_archive_data( + swhid: SWHID, raw: bool = False, recursive: bool = False +) -> None: + # Already in mock archive + if swhid in METADATA and not raw: + return + url = swhid2url(swhid) SWHID2URL[str(swhid)] = url @@ -47,19 +71,26 @@ MOCK_ARCHIVE[url] = data METADATA[swhid] = data + # Retrieve additional needed data for different artifacts (eg: content's + # blob data, revision parents, etc.) + if recursive: + if swhid.object_type == CONTENT: + generate_archive_data(swhid, raw=True) + elif swhid.object_type == REVISION: + for parent in METADATA[swhid]["parents"]: + parent_swhid = parse_swhid(f"swh:1:rev:{parent['id']}") + # Only retrieve one-level of parent (disable recursivity) + generate_archive_data(parent_swhid) + elif swhid.object_type == RELEASE: + target_type = METADATA[swhid]["target_type"] + target_id = METADATA[swhid]["target"] + target = parse_swhid(f"swh:1:{get_short_type(target_type)}:{target_id}") + generate_archive_data(target, recursive=True) + for entry in ALL_ENTRIES: swhid = parse_swhid(entry) - generate_archive_data(swhid) - - # Retrieve raw blob data for content artifact - if swhid.object_type == CONTENT: - generate_archive_data(swhid, raw=True) - # Retrieve parent commits for revision artifact - elif swhid.object_type == REVISION: - for parent in METADATA[swhid]["parents"]: - parent_swhid = parse_swhid(f"swh:1:rev:{parent['id']}") - generate_archive_data(parent_swhid) + generate_archive_data(swhid, recursive=True) print("# GENERATED FILE, DO NOT EDIT.") print("# Run './gen-api-data.py > api_data.py' instead.") diff --git a/swh/fuse/tests/test_directory.py b/swh/fuse/tests/test_directory.py --- a/swh/fuse/tests/test_directory.py +++ b/swh/fuse/tests/test_directory.py @@ -1,13 +1,12 @@ import os -from swh.fuse.tests.common import get_data_from_archive +from swh.fuse.tests.common import get_dir_name_entries from swh.fuse.tests.data.config import DIR_WITH_SUBMODULES, ROOT_DIR def test_list_dir(fuse_mntdir): dir_path = fuse_mntdir / "archive" / ROOT_DIR - dir_meta = get_data_from_archive(ROOT_DIR) - expected = [x["name"] for x in dir_meta] + expected = get_dir_name_entries(ROOT_DIR) actual = os.listdir(dir_path) assert set(actual) == set(expected) diff --git a/swh/fuse/tests/test_release.py b/swh/fuse/tests/test_release.py new file mode 100644 --- /dev/null +++ b/swh/fuse/tests/test_release.py @@ -0,0 +1,30 @@ +import json +import os + +from swh.fuse.tests.common import get_data_from_archive, get_dir_name_entries +from swh.fuse.tests.data.config import ROOT_DIR, ROOT_REL + + +def test_access_meta(fuse_mntdir): + file_path = fuse_mntdir / "archive" / ROOT_REL / "meta.json" + expected = json.dumps(get_data_from_archive(ROOT_REL)) + assert file_path.read_text() == expected + + +def test_access_target(fuse_mntdir): + target_path = fuse_mntdir / "archive" / ROOT_REL / "target" + expected = ["meta.json", "root", "parent", "parents"] + actual = os.listdir(target_path) + assert set(actual) == set(expected) + + +def test_target_type(fuse_mntdir): + file_path = fuse_mntdir / "archive" / ROOT_REL / "target_type" + assert file_path.read_text() == "revision\n" + + +def test_access_root(fuse_mntdir): + dir_path = fuse_mntdir / "archive" / ROOT_REL / "root" + expected = get_dir_name_entries(ROOT_DIR) + actual = os.listdir(dir_path) + assert set(actual) == set(expected) diff --git a/swh/fuse/tests/test_revision.py b/swh/fuse/tests/test_revision.py --- a/swh/fuse/tests/test_revision.py +++ b/swh/fuse/tests/test_revision.py @@ -1,7 +1,7 @@ import json import os -from swh.fuse.tests.common import get_data_from_archive +from swh.fuse.tests.common import get_data_from_archive, get_dir_name_entries from swh.fuse.tests.data.config import ROOT_DIR, ROOT_REV @@ -13,8 +13,7 @@ def test_list_root(fuse_mntdir): dir_path = fuse_mntdir / "archive" / ROOT_REV / "root" - dir_meta = get_data_from_archive(ROOT_DIR) - expected = [x["name"] for x in dir_meta] + expected = get_dir_name_entries(ROOT_DIR) actual = os.listdir(dir_path) assert set(actual) == set(expected)