diff --git a/swh/fuse/fs/artifact.py b/swh/fuse/fs/artifact.py --- a/swh/fuse/fs/artifact.py +++ b/swh/fuse/fs/artifact.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, AsyncIterator, List +import urllib.parse from swh.fuse.fs.entry import ( EntryMode, @@ -15,7 +16,7 @@ FuseSymlinkEntry, ) from swh.model.from_disk import DentryPerms -from swh.model.identifiers import CONTENT, DIRECTORY, RELEASE, REVISION, SWHID +from swh.model.identifiers import CONTENT, DIRECTORY, RELEASE, REVISION, SNAPSHOT, SWHID @dataclass @@ -264,9 +265,38 @@ return len(await self.get_content()) +@dataclass +class Snapshot(FuseDirEntry): + """ Software Heritage snapshot artifact. + + Attributes: + swhid: Software Heritage persistent identifier + + Snapshot nodes are represented on the file-system as directories with one + entry for each branch in the snapshot. Each entry is a symlink pointing into + `archive/` to the branch target SWHID. Branch names are URL encoded (hence + '/' are replaced with '%2F'). """ + + swhid: SWHID + + async def __aiter__(self) -> AsyncIterator[FuseEntry]: + metadata = await self.fuse.get_metadata(self.swhid) + root_path = self.get_relative_root_path() + + for branch_name, branch_meta in metadata.items(): + # Mangle branch name to create a valid UNIX filename + name = urllib.parse.quote_plus(branch_name) + yield self.create_child( + FuseSymlinkEntry, + name=name, + target=Path(root_path, f"archive/{branch_meta['target']}"), + ) + + OBJTYPE_GETTERS = { CONTENT: Content, DIRECTORY: Directory, REVISION: Revision, RELEASE: Release, + SNAPSHOT: Snapshot, } diff --git a/swh/fuse/tests/common.py b/swh/fuse/tests/common.py --- a/swh/fuse/tests/common.py +++ b/swh/fuse/tests/common.py @@ -14,7 +14,12 @@ url = SWHID2URL[swhid] if raw: url += "raw/" - return MOCK_ARCHIVE[url] + + # Special case: snapshots Web API and Web Client API differ a bit in format + if url.startswith("snapshot"): + return MOCK_ARCHIVE[url]["branches"] + else: + return MOCK_ARCHIVE[url] def get_dir_name_entries(swhid: str) -> List[str]: diff --git a/swh/fuse/tests/data/api_data.py b/swh/fuse/tests/data/api_data.py --- a/swh/fuse/tests/data/api_data.py +++ b/swh/fuse/tests/data/api_data.py @@ -1,10 +1,11 @@ # GENERATED FILE, DO NOT EDIT. # Run './gen-api-data.py > api_data.py' instead. # flake8: noqa +from typing import Any, Dict API_URL = "https://invalid-test-only.archive.softwareheritage.org/api/1" -SWHID2URL = { +SWHID2URL: Dict[str, str] = { "swh:1:cnt:61d3c9e1157203f0c4ed5165608d92294eaca808": "content/sha1_git:61d3c9e1157203f0c4ed5165608d92294eaca808/", "swh:1:dir:c6dcbe9711ea6d5a31429a833a3d0c59cbbb2578": "directory/c6dcbe9711ea6d5a31429a833a3d0c59cbbb2578/", "swh:1:dir:80ae84abc6122c47aae597fde99645f8663d1aba": "directory/80ae84abc6122c47aae597fde99645f8663d1aba/", @@ -21,9 +22,11 @@ "swh:1:cnt:be5effea679c057aec2bb020f0241b1d1d660840": "content/sha1_git:be5effea679c057aec2bb020f0241b1d1d660840/", "swh:1:rel:3a7b2dfffed2945d2933ba4ebc063adba35ddb2e": "release/3a7b2dfffed2945d2933ba4ebc063adba35ddb2e/", "swh:1:dir:b24d39c928b9c3f440f8e2ec06c78f43d28d87d6": "directory/b24d39c928b9c3f440f8e2ec06c78f43d28d87d6/", + "swh:1:snp:02db117fef22434f1658b833a756775ca6effed0": "snapshot/02db117fef22434f1658b833a756775ca6effed0/", + "swh:1:rev:430a9fd4c797c50cea26157141b2408073b2ed91": "revision/430a9fd4c797c50cea26157141b2408073b2ed91/", } -MOCK_ARCHIVE = { +MOCK_ARCHIVE: Dict[str, Any] = { "content/sha1_git:61d3c9e1157203f0c4ed5165608d92294eaca808/": { "length": 10084, "status": "visible", @@ -1317,4 +1320,215 @@ "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:d8d2804032211fec42ec197827b049d5dea40ea7/", }, ], + "snapshot/02db117fef22434f1658b833a756775ca6effed0/": { + "id": "02db117fef22434f1658b833a756775ca6effed0", + "branches": { + "refs/heads/auto": { + "target": "a0eb7a2c6da31b44718188002ac0cec12a3c86ee", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/a0eb7a2c6da31b44718188002ac0cec12a3c86ee/", + }, + "refs/heads/beta": { + "target": "c980aba9a88704717229da3c1ec02685333c0db2", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/c980aba9a88704717229da3c1ec02685333c0db2/", + }, + "refs/heads/grammer": { + "target": "d7b93216cdeb477e1af813f898096af867550338", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/d7b93216cdeb477e1af813f898096af867550338/", + }, + "refs/heads/master": { + "target": "430a9fd4c797c50cea26157141b2408073b2ed91", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/430a9fd4c797c50cea26157141b2408073b2ed91/", + }, + "refs/heads/snap-stage3": { + "target": "a5c12f4e39d32af3c951b66bd2839bc0b5a1125b", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/a5c12f4e39d32af3c951b66bd2839bc0b5a1125b/", + }, + "refs/heads/stable": { + "target": "082e4763615bdbe7b4dd3dfd6fc2210b7773edf5", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/082e4763615bdbe7b4dd3dfd6fc2210b7773edf5/", + }, + "refs/heads/tmp": { + "target": "a0eb7a2c6da31b44718188002ac0cec12a3c86ee", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/a0eb7a2c6da31b44718188002ac0cec12a3c86ee/", + }, + "refs/heads/try": { + "target": "b53c0f93eedcdedd4fd89bccc5a3a09d1c5cd23e", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/b53c0f93eedcdedd4fd89bccc5a3a09d1c5cd23e/", + }, + "refs/tags/0.1": { + "target": "16e4369fe3b5f00aa3cdc584a4e41c51c0d3ca8a", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/16e4369fe3b5f00aa3cdc584a4e41c51c0d3ca8a/", + }, + "refs/tags/0.10": { + "target": "46867cc3e4ddcbb1d359a315805de00094dacaf9", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/46867cc3e4ddcbb1d359a315805de00094dacaf9/", + }, + "refs/tags/0.11.0": { + "target": "aa1163b92de7717eb7c5eba002b4012e0574a7fe", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/aa1163b92de7717eb7c5eba002b4012e0574a7fe/", + }, + "refs/tags/0.12.0": { + "target": "ba4081a5a8573875fed17545846f6f6902c8ba8d", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/ba4081a5a8573875fed17545846f6f6902c8ba8d/", + }, + "refs/tags/0.2": { + "target": "0622a74c48a708aec28d25964f4e9b4489580bc7", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/0622a74c48a708aec28d25964f4e9b4489580bc7/", + }, + "refs/tags/0.3": { + "target": "2f32a1581f522e524009138b33b1c7049ced668d", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/2f32a1581f522e524009138b33b1c7049ced668d/", + }, + "refs/tags/0.4": { + "target": "39c0d3591e0326874b7263a621ce09ecd64f0eb2", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/39c0d3591e0326874b7263a621ce09ecd64f0eb2/", + }, + "refs/tags/0.5": { + "target": "8b98e5a296d95c5e832db0756828e5bec31c6f50", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/8b98e5a296d95c5e832db0756828e5bec31c6f50/", + }, + "refs/tags/0.6": { + "target": "00dbbd01c2aee72982b3e0f9511ae1d4428c3ba9", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/00dbbd01c2aee72982b3e0f9511ae1d4428c3ba9/", + }, + "refs/tags/0.7": { + "target": "a2db7c15ce9f586164cabb15d83fb3f6bbeb3cf5", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/a2db7c15ce9f586164cabb15d83fb3f6bbeb3cf5/", + }, + "refs/tags/0.8": { + "target": "8a4f0fa6c518eb634687abe9659601d9d2a61899", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/8a4f0fa6c518eb634687abe9659601d9d2a61899/", + }, + "refs/tags/0.9": { + "target": "7613b15fdbbb9bf770a2c731f4135886b0ff3cf0", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/7613b15fdbbb9bf770a2c731f4135886b0ff3cf0/", + }, + "refs/tags/1.0.0": { + "target": "a59de37e99060162a2674e3ff45409ac73595c0e", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/a59de37e99060162a2674e3ff45409ac73595c0e/", + }, + "refs/tags/1.0.0-alpha": { + "target": "44a287e6eb22ec3c2a687fc156813577464017f7", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/44a287e6eb22ec3c2a687fc156813577464017f7/", + }, + "refs/tags/1.0.0-alpha.2": { + "target": "522d09dfecbeca1595f25ac58c6d0178bbd21d7d", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/522d09dfecbeca1595f25ac58c6d0178bbd21d7d/", + }, + "refs/tags/1.0.0-beta": { + "target": "9854143cba679834bc4ef932858cd5303f015a0e", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/9854143cba679834bc4ef932858cd5303f015a0e/", + }, + "refs/tags/1.1.0": { + "target": "35ceea3997c79a3b7562e89b462ab76af5b86b22", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/35ceea3997c79a3b7562e89b462ab76af5b86b22/", + }, + "refs/tags/homu-tmp": { + "target": "1fe32ca12c51afcd761d9962f51a74ff0d07a591", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/1fe32ca12c51afcd761d9962f51a74ff0d07a591/", + }, + "refs/tags/release-0.1": { + "target": "16e4369fe3b5f00aa3cdc584a4e41c51c0d3ca8a", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/16e4369fe3b5f00aa3cdc584a4e41c51c0d3ca8a/", + }, + "refs/tags/release-0.2": { + "target": "0622a74c48a708aec28d25964f4e9b4489580bc7", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/0622a74c48a708aec28d25964f4e9b4489580bc7/", + }, + "refs/tags/release-0.3": { + "target": "2f32a1581f522e524009138b33b1c7049ced668d", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/2f32a1581f522e524009138b33b1c7049ced668d/", + }, + "refs/tags/release-0.3.1": { + "target": "33a055638c637d2f63d1d7a18b235c93c08d10b8", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/33a055638c637d2f63d1d7a18b235c93c08d10b8/", + }, + "refs/tags/release-0.4": { + "target": "39c0d3591e0326874b7263a621ce09ecd64f0eb2", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/39c0d3591e0326874b7263a621ce09ecd64f0eb2/", + }, + "refs/tags/release-0.5": { + "target": "8b98e5a296d95c5e832db0756828e5bec31c6f50", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/8b98e5a296d95c5e832db0756828e5bec31c6f50/", + }, + "refs/tags/release-0.6": { + "target": "00dbbd01c2aee72982b3e0f9511ae1d4428c3ba9", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/00dbbd01c2aee72982b3e0f9511ae1d4428c3ba9/", + }, + "refs/tags/release-0.7": { + "target": "a2db7c15ce9f586164cabb15d83fb3f6bbeb3cf5", + "target_type": "revision", + "target_url": "https://archive.softwareheritage.org/api/1/revision/a2db7c15ce9f586164cabb15d83fb3f6bbeb3cf5/", + }, + }, + "next_branch": None, + }, + "revision/430a9fd4c797c50cea26157141b2408073b2ed91/": { + "message": 'Auto merge of #27534 - alexcrichton:revert-adding-gdb-pp-tests, r=michaelwoerister\n\n… are not actually broken."\r\n\r\nThis reverts commit 354cf4b56b8e2af67cc68965eb816deec0e79e4b.\r\n\r\n\r\nUnfortunately these [tests are failing](http://buildbot.rust-lang.org/builders/nightly-dist-rustc-linux/builds/224/steps/distcheck/logs/stdio) on the snapshot/nightly bots with the [same message](https://gist.github.com/alexcrichton/611705ded07b0d73ded9) found in #27514\r\n', + "author": { + "fullname": "bors ", + "name": "bors", + "email": "bors@rust-lang.org", + }, + "committer": { + "fullname": "bors ", + "name": "bors", + "email": "bors@rust-lang.org", + }, + "date": "2015-08-05T18:48:53+00:00", + "committer_date": "2015-08-05T18:48:53+00:00", + "type": "git", + "directory": "1ac29db0e7280af41064676569a96d1f88ccfa96", + "synthetic": False, + "metadata": {}, + "parents": [ + { + "id": "d03456183e85fe7bd465bbe7c8f67885a2528444", + "url": "https://archive.softwareheritage.org/api/1/revision/d03456183e85fe7bd465bbe7c8f67885a2528444/", + }, + { + "id": "3430532a8ee8fd5a7f47647c8c29403384647095", + "url": "https://archive.softwareheritage.org/api/1/revision/3430532a8ee8fd5a7f47647c8c29403384647095/", + }, + ], + "id": "430a9fd4c797c50cea26157141b2408073b2ed91", + "extra_headers": [], + "merge": True, + "url": "https://archive.softwareheritage.org/api/1/revision/430a9fd4c797c50cea26157141b2408073b2ed91/", + "history_url": "https://archive.softwareheritage.org/api/1/revision/430a9fd4c797c50cea26157141b2408073b2ed91/log/", + "directory_url": "https://archive.softwareheritage.org/api/1/directory/1ac29db0e7280af41064676569a96d1f88ccfa96/", + }, } diff --git a/swh/fuse/tests/data/config.py b/swh/fuse/tests/data/config.py --- a/swh/fuse/tests/data/config.py +++ b/swh/fuse/tests/data/config.py @@ -23,9 +23,14 @@ ] # Release ROOT_REL = "swh:1:rel:874f7cbe352033cac5a8bc889847da2fe1d13e9f" -# TODO # Snapshot -# TODO +# WARNING: Do not use a snapshot artifact which is paginated because it will be +# a pain to synchronize formats properly between the mock Web API/Web API Client +# (since they differ slightly for snapshots). The Web API Client is already +# responsible for merging everything together so the real API will have no +# problem, only the mock offline one. +ROOT_SNP = "swh:1:snp:02db117fef22434f1658b833a756775ca6effed0" +ROOT_SNP_MASTER_BRANCH = "swh:1:rev:430a9fd4c797c50cea26157141b2408073b2ed91" # Special corner cases (not from Rust compiler) REL_TARGET_CNT = "swh:1:rel:da5f9898d6248ab26277116f54aca855338401d2" @@ -42,4 +47,6 @@ ROOT_REL, REL_TARGET_CNT, REL_TARGET_DIR, + ROOT_SNP, + ROOT_SNP_MASTER_BRANCH, ] diff --git a/swh/fuse/tests/data/gen-api-data.py b/swh/fuse/tests/data/gen-api-data.py --- a/swh/fuse/tests/data/gen-api-data.py +++ b/swh/fuse/tests/data/gen-api-data.py @@ -17,6 +17,7 @@ DIRECTORY, RELEASE, REVISION, + SNAPSHOT, SWHID, parse_swhid, ) @@ -36,6 +37,7 @@ DIRECTORY: "directory/", REVISION: "revision/", RELEASE: "release/", + SNAPSHOT: "snapshot/", } return f"{prefix[swhid.object_type]}{swhid.object_id}/" @@ -47,6 +49,7 @@ DIRECTORY: "dir", REVISION: "rev", RELEASE: "rel", + SNAPSHOT: "snp", } return short_type[object_type] @@ -90,7 +93,8 @@ print("# GENERATED FILE, DO NOT EDIT.") print("# Run './gen-api-data.py > api_data.py' instead.") print("# flake8: noqa") +print("from typing import Any, Dict") print("") print(f"API_URL = '{API_URL_test}'\n") -print(f"SWHID2URL = {SWHID2URL}\n") -print(f"MOCK_ARCHIVE = {MOCK_ARCHIVE}") +print(f"SWHID2URL: Dict[str, str] = {SWHID2URL}\n") +print(f"MOCK_ARCHIVE: Dict[str, Any] = {MOCK_ARCHIVE}") diff --git a/swh/fuse/tests/test_snapshot.py b/swh/fuse/tests/test_snapshot.py new file mode 100644 --- /dev/null +++ b/swh/fuse/tests/test_snapshot.py @@ -0,0 +1,22 @@ +import os +import urllib.parse + +from swh.fuse.tests.common import get_data_from_archive +from swh.fuse.tests.data.config import ROOT_SNP + + +def test_list_branches(fuse_mntdir): + snp_meta = get_data_from_archive(ROOT_SNP) + expected = snp_meta.keys() + # Mangle branch name to create a valid UNIX filename + expected = [urllib.parse.quote_plus(x) for x in expected] + actual = os.listdir(fuse_mntdir / "archive" / ROOT_SNP) + assert set(actual) == set(expected) + + +def test_access_rev_target(fuse_mntdir): + branch_name = urllib.parse.quote_plus("refs/heads/master") + dir_path = fuse_mntdir / "archive" / ROOT_SNP / branch_name + expected = ["meta.json", "root", "parent", "parents"] + actual = os.listdir(dir_path) + assert set(actual) == set(expected)