Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/in_memory.py
Show First 20 Lines • Show All 131 Lines • ▼ Show 20 Lines | def reset(self): | ||||
self._content_indexes = defaultdict(lambda: defaultdict(set)) | self._content_indexes = defaultdict(lambda: defaultdict(set)) | ||||
self._skipped_contents = {} | self._skipped_contents = {} | ||||
self._skipped_content_indexes = defaultdict(lambda: defaultdict(set)) | self._skipped_content_indexes = defaultdict(lambda: defaultdict(set)) | ||||
self._directories = {} | self._directories = {} | ||||
self._revisions = {} | self._revisions = {} | ||||
self._releases = {} | self._releases = {} | ||||
self._snapshots = {} | self._snapshots = {} | ||||
self._origins = {} | self._origins = {} | ||||
self._origins_by_id = [] | |||||
self._origins_by_sha1 = {} | self._origins_by_sha1 = {} | ||||
self._origin_visits = {} | self._origin_visits = {} | ||||
self._origin_visit_statuses: Dict[Tuple[str, int], List[OriginVisitStatus]] = {} | self._origin_visit_statuses: Dict[Tuple[str, int], List[OriginVisitStatus]] = {} | ||||
self._persons = {} | self._persons = {} | ||||
# {object_type: {id: {authority: [metadata]}}} | # {object_type: {id: {authority: [metadata]}}} | ||||
self._raw_extrinsic_metadata: Dict[ | self._raw_extrinsic_metadata: Dict[ | ||||
MetadataTargetType, | MetadataTargetType, | ||||
▲ Show 20 Lines • Show All 527 Lines • ▼ Show 20 Lines | def origin_get_one(self, origin_url: str) -> Optional[Origin]: | ||||
return self._origins.get(origin_url) | return self._origins.get(origin_url) | ||||
def origin_get(self, origins: List[str]) -> Iterable[Optional[Origin]]: | def origin_get(self, origins: List[str]) -> Iterable[Optional[Origin]]: | ||||
return [self.origin_get_one(origin_url) for origin_url in origins] | return [self.origin_get_one(origin_url) for origin_url in origins] | ||||
def origin_get_by_sha1(self, sha1s): | def origin_get_by_sha1(self, sha1s): | ||||
return [self._convert_origin(self._origins_by_sha1.get(sha1)) for sha1 in sha1s] | return [self._convert_origin(self._origins_by_sha1.get(sha1)) for sha1 in sha1s] | ||||
def origin_get_range(self, origin_from=1, origin_count=100): | |||||
origin_from = max(origin_from, 1) | |||||
if origin_from <= len(self._origins_by_id): | |||||
max_idx = origin_from + origin_count - 1 | |||||
if max_idx > len(self._origins_by_id): | |||||
max_idx = len(self._origins_by_id) | |||||
for idx in range(origin_from - 1, max_idx): | |||||
origin = self._convert_origin(self._origins[self._origins_by_id[idx]]) | |||||
yield {"id": idx + 1, **origin} | |||||
def origin_list( | def origin_list( | ||||
self, page_token: Optional[str] = None, limit: int = 100 | self, page_token: Optional[str] = None, limit: int = 100 | ||||
) -> PagedResult[Origin]: | ) -> PagedResult[Origin]: | ||||
origin_urls = sorted(self._origins) | origin_urls = sorted(self._origins) | ||||
from_ = bisect.bisect_left(origin_urls, page_token) if page_token else 0 | from_ = bisect.bisect_left(origin_urls, page_token) if page_token else 0 | ||||
next_page_token = None | next_page_token = None | ||||
# Take one more origin so we can reuse it as the next page token if any | # Take one more origin so we can reuse it as the next page token if any | ||||
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | def origin_add(self, origins: List[Origin]) -> Dict[str, int]: | ||||
self.origin_add_one(origin) | self.origin_add_one(origin) | ||||
added += 1 | added += 1 | ||||
return {"origin:add": added} | return {"origin:add": added} | ||||
def origin_add_one(self, origin: Origin) -> str: | def origin_add_one(self, origin: Origin) -> str: | ||||
if origin.url not in self._origins: | if origin.url not in self._origins: | ||||
self.journal_writer.origin_add([origin]) | self.journal_writer.origin_add([origin]) | ||||
# generate an origin_id because it is needed by origin_get_range. | |||||
# TODO: remove this when we remove origin_get_range | |||||
origin_id = len(self._origins) + 1 | |||||
self._origins_by_id.append(origin.url) | |||||
assert len(self._origins_by_id) == origin_id | |||||
self._origins[origin.url] = origin | self._origins[origin.url] = origin | ||||
self._origins_by_sha1[origin_url_to_sha1(origin.url)] = origin | self._origins_by_sha1[origin_url_to_sha1(origin.url)] = origin | ||||
self._origin_visits[origin.url] = [] | self._origin_visits[origin.url] = [] | ||||
self._objects[origin.url].append(("origin", origin.url)) | self._objects[origin.url].append(("origin", origin.url)) | ||||
return origin.url | return origin.url | ||||
def origin_visit_add(self, visits: List[OriginVisit]) -> Iterable[OriginVisit]: | def origin_visit_add(self, visits: List[OriginVisit]) -> Iterable[OriginVisit]: | ||||
▲ Show 20 Lines • Show All 493 Lines • Show Last 20 Lines |