Changeset View
Changeset View
Standalone View
Standalone View
swh/fuse/fs/mountpoint.py
| Show All 25 Lines | async def __aiter__(self) -> AsyncIterator[FuseEntry]: | ||||
| yield self.create_child(MetaDir) | yield self.create_child(MetaDir) | ||||
| @dataclass | @dataclass | ||||
| class ArchiveDir(FuseDirEntry): | class ArchiveDir(FuseDirEntry): | ||||
| """ The archive/ directory is lazily populated with one entry per accessed | """ The archive/ directory is lazily populated with one entry per accessed | ||||
| SWHID, having actual SWHIDs as names """ | SWHID, having actual SWHIDs as names """ | ||||
| prefix: str = field(default="") | |||||
| name: str = field(init=False, default="archive") | name: str = field(init=False, default="archive") | ||||
| mode: int = field(init=False, default=int(EntryMode.RDONLY_DIR)) | mode: int = field(init=False, default=int(EntryMode.RDONLY_DIR)) | ||||
| def __post_init__(self): | |||||
| super().__post_init__() | |||||
| if self.prefix: | |||||
| self.name = self.prefix[-self.fuse.conf["sharding"]["length"] :] | |||||
| def create_child(self, swhid: SWHID) -> FuseEntry: | def create_child(self, swhid: SWHID) -> FuseEntry: | ||||
| if swhid.object_type == CONTENT: | if swhid.object_type == CONTENT: | ||||
| mode = EntryMode.RDONLY_FILE | mode = EntryMode.RDONLY_FILE | ||||
| else: | else: | ||||
| mode = EntryMode.RDONLY_DIR | mode = EntryMode.RDONLY_DIR | ||||
| return super().create_child( | return super().create_child( | ||||
| OBJTYPE_GETTERS[swhid.object_type], | OBJTYPE_GETTERS[swhid.object_type], | ||||
| name=str(swhid), | name=str(swhid), | ||||
| mode=int(mode), | mode=int(mode), | ||||
| swhid=swhid, | swhid=swhid, | ||||
| ) | ) | ||||
| async def __aiter__(self) -> AsyncIterator[FuseEntry]: | async def __aiter__(self) -> AsyncIterator[FuseEntry]: | ||||
| entries = [] | |||||
| async for swhid in self.fuse.cache.get_cached_swhids(): | async for swhid in self.fuse.cache.get_cached_swhids(): | ||||
| if swhid.object_id.startswith(self.prefix): | |||||
| entries.append(swhid) | |||||
| sharding_depth = len(self.prefix) // self.fuse.conf["sharding"]["length"] | |||||
| if sharding_depth < self.fuse.conf["sharding"]["depth"]: | |||||
| # Sharding intermediate directories | |||||
| for prefix in self.get_sharded_next_prefixes(self.prefix, entries): | |||||
| yield super().create_child(ArchiveDir, prefix=self.prefix + prefix) | |||||
| else: | |||||
| # Real archive entries | |||||
| for swhid in entries: | |||||
| yield self.create_child(swhid) | yield self.create_child(swhid) | ||||
| async def lookup(self, name: str) -> FuseEntry: | async def lookup(self, name: str) -> FuseEntry: | ||||
| entry = await super().lookup(name) | entry = await super().lookup(name) | ||||
| if entry: | if entry: | ||||
| return entry | return entry | ||||
| # On the fly mounting of a new artifact | |||||
| try: | try: | ||||
| # On the fly mounting of a new artifact | |||||
| swhid = parse_swhid(name) | swhid = parse_swhid(name) | ||||
| await self.fuse.get_metadata(swhid) | await self.fuse.get_metadata(swhid) | ||||
| return self.create_child(swhid) | return self.create_child(swhid) | ||||
| except ValidationError: | except ValidationError: | ||||
| return None | # Sharded intermediate directory | ||||
| return super().create_child(ArchiveDir, prefix=self.prefix + name) | |||||
| @dataclass | @dataclass | ||||
| class MetaDir(FuseDirEntry): | class MetaDir(FuseDirEntry): | ||||
| """ The meta/ directory contains one SWHID.json file for each SWHID entry | """ The meta/ directory contains one SWHID.json file for each SWHID entry | ||||
| under archive/. The JSON file contain all available meta information about | under archive/. The JSON file contain all available meta information about | ||||
| the given SWHID, as returned by the Software Heritage Web API for that | the given SWHID, as returned by the Software Heritage Web API for that | ||||
| object. Note that, in case of pagination (e.g., snapshot objects with many | object. Note that, in case of pagination (e.g., snapshot objects with many | ||||
| branches) the JSON file will contain a complete version with all pages | branches) the JSON file will contain a complete version with all pages | ||||
| merged together. """ | merged together. """ | ||||
| prefix: str = field(default="") | |||||
| name: str = field(init=False, default="meta") | name: str = field(init=False, default="meta") | ||||
| mode: int = field(init=False, default=int(EntryMode.RDONLY_DIR)) | mode: int = field(init=False, default=int(EntryMode.RDONLY_DIR)) | ||||
| def __post_init__(self): | |||||
| super().__post_init__() | |||||
| if self.prefix: | |||||
| self.name = self.prefix[-self.fuse.conf["sharding"]["length"] :] | |||||
| async def __aiter__(self) -> AsyncIterator[FuseEntry]: | async def __aiter__(self) -> AsyncIterator[FuseEntry]: | ||||
| entries = [] | |||||
| async for swhid in self.fuse.cache.get_cached_swhids(): | async for swhid in self.fuse.cache.get_cached_swhids(): | ||||
| if swhid.object_id.startswith(self.prefix): | |||||
| entries.append(swhid) | |||||
| sharding_depth = len(self.prefix) // self.fuse.conf["sharding"]["length"] | |||||
| if sharding_depth < self.fuse.conf["sharding"]["depth"]: | |||||
| # Sharding intermediate directories | |||||
| for prefix in self.get_sharded_next_prefixes(self.prefix, entries): | |||||
| yield self.create_child(MetaDir, prefix=self.prefix + prefix) | |||||
| else: | |||||
| # Real meta entries | |||||
| for swhid in entries: | |||||
| yield self.create_child( | yield self.create_child( | ||||
| MetaEntry, | MetaEntry, | ||||
| name=f"{swhid}.json", | name=f"{swhid}.json", | ||||
| mode=int(EntryMode.RDONLY_FILE), | mode=int(EntryMode.RDONLY_FILE), | ||||
| swhid=swhid, | swhid=swhid, | ||||
| ) | ) | ||||
| async def lookup(self, name: str) -> FuseEntry: | |||||
| entry = await super().lookup(name) | |||||
| if entry: | |||||
| return entry | |||||
| # Sharded intermediate directory | |||||
| return super().create_child(MetaDir, prefix=self.prefix + name) | |||||
| @dataclass | @dataclass | ||||
| class MetaEntry(FuseFileEntry): | class MetaEntry(FuseFileEntry): | ||||
| """ An entry from the meta/ directory, containing for each accessed SWHID a | """ An entry from the meta/ directory, containing for each accessed SWHID a | ||||
| corresponding SWHID.json file with all the metadata from the Software | corresponding SWHID.json file with all the metadata from the Software | ||||
| Heritage archive. """ | Heritage archive. """ | ||||
| swhid: SWHID | swhid: SWHID | ||||
| async def get_content(self) -> bytes: | async def get_content(self) -> bytes: | ||||
| # Get raw JSON metadata from API (un-typified) | # Get raw JSON metadata from API (un-typified) | ||||
| metadata = await self.fuse.cache.metadata.get(self.swhid, typify=False) | metadata = await self.fuse.cache.metadata.get(self.swhid, typify=False) | ||||
| return json.dumps(metadata).encode() | return json.dumps(metadata).encode() | ||||
| async def size(self) -> int: | async def size(self) -> int: | ||||
| return len(await self.get_content()) | return len(await self.get_content()) | ||||