Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/in_memory.py
| Show First 20 Lines • Show All 131 Lines • ▼ Show 20 Lines | def reset(self): | ||||
| self._content_indexes = defaultdict(lambda: defaultdict(set)) | self._content_indexes = defaultdict(lambda: defaultdict(set)) | ||||
| self._skipped_contents = {} | self._skipped_contents = {} | ||||
| self._skipped_content_indexes = defaultdict(lambda: defaultdict(set)) | self._skipped_content_indexes = defaultdict(lambda: defaultdict(set)) | ||||
| self._directories = {} | self._directories = {} | ||||
| self._revisions = {} | self._revisions = {} | ||||
| self._releases = {} | self._releases = {} | ||||
| self._snapshots = {} | self._snapshots = {} | ||||
| self._origins = {} | self._origins = {} | ||||
| self._origins_by_id = [] | |||||
| self._origins_by_sha1 = {} | self._origins_by_sha1 = {} | ||||
| self._origin_visits = {} | self._origin_visits = {} | ||||
| self._origin_visit_statuses: Dict[Tuple[str, int], List[OriginVisitStatus]] = {} | self._origin_visit_statuses: Dict[Tuple[str, int], List[OriginVisitStatus]] = {} | ||||
| self._persons = {} | self._persons = {} | ||||
| # {object_type: {id: {authority: [metadata]}}} | # {object_type: {id: {authority: [metadata]}}} | ||||
| self._raw_extrinsic_metadata: Dict[ | self._raw_extrinsic_metadata: Dict[ | ||||
| MetadataTargetType, | MetadataTargetType, | ||||
| ▲ Show 20 Lines • Show All 527 Lines • ▼ Show 20 Lines | def origin_get_one(self, origin_url: str) -> Optional[Origin]: | ||||
| return self._origins.get(origin_url) | return self._origins.get(origin_url) | ||||
| def origin_get(self, origins: List[str]) -> Iterable[Optional[Origin]]: | def origin_get(self, origins: List[str]) -> Iterable[Optional[Origin]]: | ||||
| return [self.origin_get_one(origin_url) for origin_url in origins] | return [self.origin_get_one(origin_url) for origin_url in origins] | ||||
| def origin_get_by_sha1(self, sha1s): | def origin_get_by_sha1(self, sha1s): | ||||
| return [self._convert_origin(self._origins_by_sha1.get(sha1)) for sha1 in sha1s] | return [self._convert_origin(self._origins_by_sha1.get(sha1)) for sha1 in sha1s] | ||||
| def origin_get_range(self, origin_from=1, origin_count=100): | |||||
| origin_from = max(origin_from, 1) | |||||
| if origin_from <= len(self._origins_by_id): | |||||
| max_idx = origin_from + origin_count - 1 | |||||
| if max_idx > len(self._origins_by_id): | |||||
| max_idx = len(self._origins_by_id) | |||||
| for idx in range(origin_from - 1, max_idx): | |||||
| origin = self._convert_origin(self._origins[self._origins_by_id[idx]]) | |||||
| yield {"id": idx + 1, **origin} | |||||
| def origin_list( | def origin_list( | ||||
| self, page_token: Optional[str] = None, limit: int = 100 | self, page_token: Optional[str] = None, limit: int = 100 | ||||
| ) -> PagedResult[Origin]: | ) -> PagedResult[Origin]: | ||||
| origin_urls = sorted(self._origins) | origin_urls = sorted(self._origins) | ||||
| from_ = bisect.bisect_left(origin_urls, page_token) if page_token else 0 | from_ = bisect.bisect_left(origin_urls, page_token) if page_token else 0 | ||||
| next_page_token = None | next_page_token = None | ||||
| # Take one more origin so we can reuse it as the next page token if any | # Take one more origin so we can reuse it as the next page token if any | ||||
| ▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | def origin_add(self, origins: List[Origin]) -> Dict[str, int]: | ||||
| self.origin_add_one(origin) | self.origin_add_one(origin) | ||||
| added += 1 | added += 1 | ||||
| return {"origin:add": added} | return {"origin:add": added} | ||||
| def origin_add_one(self, origin: Origin) -> str: | def origin_add_one(self, origin: Origin) -> str: | ||||
| if origin.url not in self._origins: | if origin.url not in self._origins: | ||||
| self.journal_writer.origin_add([origin]) | self.journal_writer.origin_add([origin]) | ||||
| # generate an origin_id because it is needed by origin_get_range. | |||||
| # TODO: remove this when we remove origin_get_range | |||||
| origin_id = len(self._origins) + 1 | |||||
| self._origins_by_id.append(origin.url) | |||||
| assert len(self._origins_by_id) == origin_id | |||||
| self._origins[origin.url] = origin | self._origins[origin.url] = origin | ||||
| self._origins_by_sha1[origin_url_to_sha1(origin.url)] = origin | self._origins_by_sha1[origin_url_to_sha1(origin.url)] = origin | ||||
| self._origin_visits[origin.url] = [] | self._origin_visits[origin.url] = [] | ||||
| self._objects[origin.url].append(("origin", origin.url)) | self._objects[origin.url].append(("origin", origin.url)) | ||||
| return origin.url | return origin.url | ||||
| def origin_visit_add(self, visits: List[OriginVisit]) -> Iterable[OriginVisit]: | def origin_visit_add(self, visits: List[OriginVisit]) -> Iterable[OriginVisit]: | ||||
| ▲ Show 20 Lines • Show All 493 Lines • Show Last 20 Lines | |||||