Differential D3740 Diff 13183 swh/storage/cassandra/storage.py

Changeset View

Standalone View

swh/storage/cassandra/storage.py

Show All 20 Lines	from swh.model.model import (
Release,		Release,
Directory,		Directory,
DirectoryEntry,		DirectoryEntry,
Content,		Content,
SkippedContent,		SkippedContent,
OriginVisit,		OriginVisit,
OriginVisitStatus,		OriginVisitStatus,
Snapshot,		Snapshot,
		SnapshotBranch,
		TargetType,
Origin,		Origin,
MetadataAuthority,		MetadataAuthority,
MetadataAuthorityType,		MetadataAuthorityType,
MetadataFetcher,		MetadataFetcher,
MetadataTargetType,		MetadataTargetType,
RawExtrinsicMetadata,		RawExtrinsicMetadata,
Sha1Git,		Sha1Git,
)		)
from swh.storage.interface import ListOrder, PagedResult, Sha1, VISIT_STATUSES		from swh.storage.interface import (
		ListOrder,
		PagedResult,
		PartialBranches,
		Sha1,
		VISIT_STATUSES,
		)
from swh.storage.objstorage import ObjStorage		from swh.storage.objstorage import ObjStorage
from swh.storage.writer import JournalWriter		from swh.storage.writer import JournalWriter
from swh.storage.utils import map_optional, now		from swh.storage.utils import map_optional, now

from ..exc import StorageArgumentException, HashCollision		from ..exc import StorageArgumentException, HashCollision
from .common import TOKEN_BEGIN, TOKEN_END		from .common import TOKEN_BEGIN, TOKEN_END
from . import converters		from . import converters
from .cql import CqlRunner		from .cql import CqlRunner
▲ Show 20 Lines • Show All 561 Lines • ▼ Show 20 Lines	def snapshot_add(self, snapshots: List[Snapshot]) -> Dict:
self._cql_runner.snapshot_add_one(snapshot.id)		self._cql_runner.snapshot_add_one(snapshot.id)

return {"snapshot:add": len(snapshots)}		return {"snapshot:add": len(snapshots)}

def snapshot_missing(self, snapshots: List[Sha1Git]) -> Iterable[Sha1Git]:		def snapshot_missing(self, snapshots: List[Sha1Git]) -> Iterable[Sha1Git]:
return self._cql_runner.snapshot_missing(snapshots)		return self._cql_runner.snapshot_missing(snapshots)

def snapshot_get(self, snapshot_id: Sha1Git) -> Optional[Dict[str, Any]]:		def snapshot_get(self, snapshot_id: Sha1Git) -> Optional[Dict[str, Any]]:
return self.snapshot_get_branches(snapshot_id)		d = self.snapshot_get_branches(snapshot_id)
		if d is None:
		return None
		return {
		"id": d["id"],
		"branches": {
		name: branch.to_dict() if branch else None
		for (name, branch) in d["branches"].items()
		},
		"next_branch": d["next_branch"],
		}

def snapshot_get_by_origin_visit(		def snapshot_get_by_origin_visit(
self, origin: str, visit: int		self, origin: str, visit: int
) -> Optional[Dict[str, Any]]:		) -> Optional[Dict[str, Any]]:
visit_status = self.origin_visit_status_get_latest(		visit_status = self.origin_visit_status_get_latest(
origin, visit, require_snapshot=True		origin, visit, require_snapshot=True
)		)
if visit_status and visit_status.snapshot:		if visit_status and visit_status.snapshot:
Show All 14 Lines	def snapshot_count_branches(self, snapshot_id: Sha1Git) -> Optional[Dict[str, int]]:
return counts		return counts

def snapshot_get_branches(		def snapshot_get_branches(
self,		self,
snapshot_id: Sha1Git,		snapshot_id: Sha1Git,
branches_from: bytes = b"",		branches_from: bytes = b"",
branches_count: int = 1000,		branches_count: int = 1000,
target_types: Optional[List[str]] = None,		target_types: Optional[List[str]] = None,
) -> Optional[Dict[str, Any]]:		) -> Optional[PartialBranches]:
if self._cql_runner.snapshot_missing([snapshot_id]):		if self._cql_runner.snapshot_missing([snapshot_id]):
# Makes sure we don't fetch branches for a snapshot that is		# Makes sure we don't fetch branches for a snapshot that is
# being added.		# being added.
return None		return None

branches: List = []		branches: List = []
while len(branches) < branches_count + 1:		while len(branches) < branches_count + 1:
new_branches = list(		new_branches = list(
Show All 22 Lines	) -> Optional[PartialBranches]:
if len(new_branches) < branches_count + 1:		if len(new_branches) < branches_count + 1:
break		break

if len(branches) > branches_count:		if len(branches) > branches_count:
last_branch = branches.pop(-1).name		last_branch = branches.pop(-1).name
else:		else:
last_branch = None		last_branch = None

branches_d = {		return PartialBranches(
branch.name: {"target": branch.target, "target_type": branch.target_type,}		id=snapshot_id,
if branch.target		branches={
else None		branch.name: None
		if branch.target is None
		else SnapshotBranch(
		target=branch.target, target_type=TargetType(branch.target_type)
		)
for branch in branches		for branch in branches
}		},
		next_branch=last_branch,
return {		)
"id": snapshot_id,
"branches": branches_d,
"next_branch": last_branch,
}

def snapshot_get_random(self) -> Sha1Git:		def snapshot_get_random(self) -> Sha1Git:
return self._cql_runner.snapshot_get_random().id		return self._cql_runner.snapshot_get_random().id

def object_find_by_sha1_git(self, ids: List[Sha1Git]) -> Dict[Sha1Git, List[Dict]]:		def object_find_by_sha1_git(self, ids: List[Sha1Git]) -> Dict[Sha1Git, List[Dict]]:
results: Dict[Sha1Git, List[Dict]] = {id_: [] for id_ in ids}		results: Dict[Sha1Git, List[Dict]] = {id_: [] for id_ in ids}
missing_ids = set(ids)		missing_ids = set(ids)

▲ Show 20 Lines • Show All 564 Lines • Show Last 20 Lines