Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/in_memory.py
| Show First 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | from swh.model.model import ( | ||||
| OriginVisitStatus, | OriginVisitStatus, | ||||
| Origin, | Origin, | ||||
| SHA1_SIZE, | SHA1_SIZE, | ||||
| MetadataAuthority, | MetadataAuthority, | ||||
| MetadataAuthorityType, | MetadataAuthorityType, | ||||
| MetadataFetcher, | MetadataFetcher, | ||||
| MetadataTargetType, | MetadataTargetType, | ||||
| RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
| Sha1Git, | |||||
| ) | ) | ||||
| from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex | from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex | ||||
| from swh.storage.interface import ListOrder, PagedResult, VISIT_STATUSES | from swh.storage.interface import ListOrder, PagedResult, VISIT_STATUSES | ||||
| from swh.storage.objstorage import ObjStorage | from swh.storage.objstorage import ObjStorage | ||||
| from swh.storage.utils import now | from swh.storage.utils import now | ||||
| from .converters import origin_url_to_sha1 | from .converters import origin_url_to_sha1 | ||||
| from .exc import StorageArgumentException, HashCollision | from .exc import StorageArgumentException, HashCollision | ||||
| ▲ Show 20 Lines • Show All 295 Lines • ▼ Show 20 Lines | def content_missing_per_sha1(self, contents): | ||||
| if content not in self._content_indexes["sha1"]: | if content not in self._content_indexes["sha1"]: | ||||
| yield content | yield content | ||||
| def content_missing_per_sha1_git(self, contents): | def content_missing_per_sha1_git(self, contents): | ||||
| for content in contents: | for content in contents: | ||||
| if content not in self._content_indexes["sha1_git"]: | if content not in self._content_indexes["sha1_git"]: | ||||
| yield content | yield content | ||||
| def content_get_random(self): | def content_get_random(self) -> Sha1Git: | ||||
| return random.choice(list(self._content_indexes["sha1_git"])) | return random.choice(list(self._content_indexes["sha1_git"])) | ||||
| def _skipped_content_add(self, contents: List[SkippedContent]) -> Dict: | def _skipped_content_add(self, contents: List[SkippedContent]) -> Dict: | ||||
| self.journal_writer.skipped_content_add(contents) | self.journal_writer.skipped_content_add(contents) | ||||
| summary = {"skipped_content:add": 0} | summary = {"skipped_content:add": 0} | ||||
| missing_contents = self.skipped_content_missing([c.hashes() for c in contents]) | missing_contents = self.skipped_content_missing([c.hashes() for c in contents]) | ||||
| ▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines | def _directory_ls(self, directory_id, recursive, prefix=b""): | ||||
| ) | ) | ||||
| def directory_ls(self, directory, recursive=False): | def directory_ls(self, directory, recursive=False): | ||||
| yield from self._directory_ls(directory, recursive) | yield from self._directory_ls(directory, recursive) | ||||
| def directory_entry_get_by_path(self, directory, paths): | def directory_entry_get_by_path(self, directory, paths): | ||||
| return self._directory_entry_get_by_path(directory, paths, b"") | return self._directory_entry_get_by_path(directory, paths, b"") | ||||
| def directory_get_random(self): | def directory_get_random(self) -> Sha1Git: | ||||
| if not self._directories: | |||||
| return None | |||||
| return random.choice(list(self._directories)) | return random.choice(list(self._directories)) | ||||
| def _directory_entry_get_by_path(self, directory, paths, prefix): | def _directory_entry_get_by_path(self, directory, paths, prefix): | ||||
| if not paths: | if not paths: | ||||
| return | return | ||||
| contents = list(self.directory_ls(directory)) | contents = list(self.directory_ls(directory)) | ||||
| ▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | def revision_log(self, revisions, limit=None): | ||||
| for rev_id in revisions: | for rev_id in revisions: | ||||
| yield from self._get_parent_revs(rev_id, seen, limit) | yield from self._get_parent_revs(rev_id, seen, limit) | ||||
| def revision_shortlog(self, revisions, limit=None): | def revision_shortlog(self, revisions, limit=None): | ||||
| yield from ( | yield from ( | ||||
| (rev["id"], rev["parents"]) for rev in self.revision_log(revisions, limit) | (rev["id"], rev["parents"]) for rev in self.revision_log(revisions, limit) | ||||
| ) | ) | ||||
| def revision_get_random(self): | def revision_get_random(self) -> Sha1Git: | ||||
| return random.choice(list(self._revisions)) | return random.choice(list(self._revisions)) | ||||
| def release_add(self, releases: List[Release]) -> Dict: | def release_add(self, releases: List[Release]) -> Dict: | ||||
| to_add = [] | to_add = [] | ||||
| for rel in releases: | for rel in releases: | ||||
| if rel.id not in self._releases and rel not in to_add: | if rel.id not in self._releases and rel not in to_add: | ||||
| to_add.append(rel) | to_add.append(rel) | ||||
| self.journal_writer.release_add(to_add) | self.journal_writer.release_add(to_add) | ||||
| Show All 11 Lines | class InMemoryStorage: | ||||
| def release_get(self, releases): | def release_get(self, releases): | ||||
| for rel_id in releases: | for rel_id in releases: | ||||
| if rel_id in self._releases: | if rel_id in self._releases: | ||||
| yield self._releases[rel_id].to_dict() | yield self._releases[rel_id].to_dict() | ||||
| else: | else: | ||||
| yield None | yield None | ||||
| def release_get_random(self): | def release_get_random(self) -> Sha1Git: | ||||
| return random.choice(list(self._releases)) | return random.choice(list(self._releases)) | ||||
| def snapshot_add(self, snapshots: List[Snapshot]) -> Dict: | def snapshot_add(self, snapshots: List[Snapshot]) -> Dict: | ||||
| count = 0 | count = 0 | ||||
| snapshots = [snap for snap in snapshots if snap.id not in self._snapshots] | snapshots = [snap for snap in snapshots if snap.id not in self._snapshots] | ||||
| for snapshot in snapshots: | for snapshot in snapshots: | ||||
| self.journal_writer.snapshot_add([snapshot]) | self.journal_writer.snapshot_add([snapshot]) | ||||
| self._snapshots[snapshot.id] = snapshot | self._snapshots[snapshot.id] = snapshot | ||||
| ▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines | ): | ||||
| } | } | ||||
| return { | return { | ||||
| "id": snapshot_id, | "id": snapshot_id, | ||||
| "branches": branches, | "branches": branches, | ||||
| "next_branch": next_branch, | "next_branch": next_branch, | ||||
| } | } | ||||
| def snapshot_get_random(self): | def snapshot_get_random(self) -> Sha1Git: | ||||
| return random.choice(list(self._snapshots)) | return random.choice(list(self._snapshots)) | ||||
| def object_find_by_sha1_git(self, ids): | def object_find_by_sha1_git(self, ids): | ||||
| ret = {} | ret = {} | ||||
| for id_ in ids: | for id_ in ids: | ||||
| objs = self._objects.get(id_, []) | objs = self._objects.get(id_, []) | ||||
| ret[id_] = [{"sha1_git": id_, "type": obj[0],} for obj in objs] | ret[id_] = [{"sha1_git": id_, "type": obj[0],} for obj in objs] | ||||
| return ret | return ret | ||||
| ▲ Show 20 Lines • Show All 602 Lines • Show Last 20 Lines | |||||