Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/storage.py
Show All 34 Lines | from swh.model.model import ( | ||||
OriginVisit, | OriginVisit, | ||||
OriginVisitStatus, | OriginVisitStatus, | ||||
Revision, | Revision, | ||||
Release, | Release, | ||||
SkippedContent, | SkippedContent, | ||||
Sha1, | Sha1, | ||||
Sha1Git, | Sha1Git, | ||||
Snapshot, | Snapshot, | ||||
SnapshotBranch, | |||||
TargetType, | |||||
SHA1_SIZE, | SHA1_SIZE, | ||||
MetadataAuthority, | MetadataAuthority, | ||||
MetadataAuthorityType, | MetadataAuthorityType, | ||||
MetadataFetcher, | MetadataFetcher, | ||||
MetadataTargetType, | MetadataTargetType, | ||||
RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
) | ) | ||||
from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex | from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex | ||||
from swh.storage.interface import ListOrder, PagedResult, VISIT_STATUSES | from swh.storage.interface import ( | ||||
ListOrder, | |||||
PagedResult, | |||||
PartialBranches, | |||||
VISIT_STATUSES, | |||||
) | |||||
from swh.storage.objstorage import ObjStorage | from swh.storage.objstorage import ObjStorage | ||||
from swh.storage.utils import now | from swh.storage.utils import now | ||||
from . import converters | from . import converters | ||||
from .common import db_transaction_generator, db_transaction | from .common import db_transaction_generator, db_transaction | ||||
from .db import Db | from .db import Db | ||||
from .exc import StorageArgumentException, StorageDBError, HashCollision | from .exc import StorageArgumentException, StorageDBError, HashCollision | ||||
from .algos import diff | from .algos import diff | ||||
▲ Show 20 Lines • Show All 671 Lines • ▼ Show 20 Lines | ) -> Iterable[Sha1Git]: | ||||
for obj in db.snapshot_missing_from_list(snapshots, cur): | for obj in db.snapshot_missing_from_list(snapshots, cur): | ||||
yield obj[0] | yield obj[0] | ||||
@timed | @timed | ||||
@db_transaction(statement_timeout=2000) | @db_transaction(statement_timeout=2000) | ||||
def snapshot_get( | def snapshot_get( | ||||
self, snapshot_id: Sha1Git, db=None, cur=None | self, snapshot_id: Sha1Git, db=None, cur=None | ||||
) -> Optional[Dict[str, Any]]: | ) -> Optional[Dict[str, Any]]: | ||||
return self.snapshot_get_branches(snapshot_id, db=db, cur=cur) | d = self.snapshot_get_branches(snapshot_id) | ||||
return { | |||||
"id": d["id"], | |||||
"branches": { | |||||
name: branch.to_dict() if branch else None | |||||
for (name, branch) in d["branches"].items() | |||||
}, | |||||
"next_branch": d["next_branch"], | |||||
} | |||||
@timed | @timed | ||||
@db_transaction(statement_timeout=2000) | @db_transaction(statement_timeout=2000) | ||||
def snapshot_get_by_origin_visit( | def snapshot_get_by_origin_visit( | ||||
self, origin: str, visit: int, db=None, cur=None | self, origin: str, visit: int, db=None, cur=None | ||||
) -> Optional[Dict[str, Any]]: | ) -> Optional[Dict[str, Any]]: | ||||
snapshot_id = db.snapshot_get_by_origin_visit(origin, visit, cur) | snapshot_id = db.snapshot_get_by_origin_visit(origin, visit, cur) | ||||
Show All 14 Lines | class Storage: | ||||
def snapshot_get_branches( | def snapshot_get_branches( | ||||
self, | self, | ||||
snapshot_id: Sha1Git, | snapshot_id: Sha1Git, | ||||
branches_from: bytes = b"", | branches_from: bytes = b"", | ||||
branches_count: int = 1000, | branches_count: int = 1000, | ||||
target_types: Optional[List[str]] = None, | target_types: Optional[List[str]] = None, | ||||
db=None, | db=None, | ||||
cur=None, | cur=None, | ||||
) -> Optional[Dict[str, Any]]: | ) -> Optional[PartialBranches]: | ||||
if snapshot_id == EMPTY_SNAPSHOT_ID: | if snapshot_id == EMPTY_SNAPSHOT_ID: | ||||
return { | return PartialBranches(id=snapshot_id, branches={}, next_branch=None,) | ||||
"id": snapshot_id, | |||||
"branches": {}, | |||||
"next_branch": None, | |||||
} | |||||
branches = {} | branches = {} | ||||
next_branch = None | next_branch = None | ||||
fetched_branches = list( | fetched_branches = list( | ||||
db.snapshot_get_by_id( | db.snapshot_get_by_id( | ||||
snapshot_id, | snapshot_id, | ||||
branches_from=branches_from, | branches_from=branches_from, | ||||
branches_count=branches_count + 1, | branches_count=branches_count + 1, | ||||
target_types=target_types, | target_types=target_types, | ||||
cur=cur, | cur=cur, | ||||
) | ) | ||||
) | ) | ||||
for branch in fetched_branches[:branches_count]: | for row in fetched_branches[:branches_count]: | ||||
branch = dict(zip(db.snapshot_get_cols, branch)) | branch_d = dict(zip(db.snapshot_get_cols, row)) | ||||
del branch["snapshot_id"] | del branch_d["snapshot_id"] | ||||
name = branch.pop("name") | name = branch_d.pop("name") | ||||
if branch == {"target": None, "target_type": None}: | if branch_d["target"] is None and branch_d["target_type"] is None: | ||||
branch = None | branch = None | ||||
else: | |||||
assert branch_d["target_type"] is not None | |||||
branch = SnapshotBranch( | |||||
target=branch_d["target"], | |||||
target_type=TargetType(branch_d["target_type"]), | |||||
) | |||||
branches[name] = branch | branches[name] = branch | ||||
if len(fetched_branches) > branches_count: | if len(fetched_branches) > branches_count: | ||||
branch = dict(zip(db.snapshot_get_cols, fetched_branches[-1])) | next_branch = dict(zip(db.snapshot_get_cols, fetched_branches[-1]))["name"] | ||||
next_branch = branch["name"] | |||||
if branches: | if branches: | ||||
return { | return PartialBranches( | ||||
"id": snapshot_id, | id=snapshot_id, branches=branches, next_branch=next_branch, | ||||
"branches": branches, | ) | ||||
"next_branch": next_branch, | |||||
} | |||||
return None | return None | ||||
@timed | @timed | ||||
@db_transaction() | @db_transaction() | ||||
def snapshot_get_random(self, db=None, cur=None) -> Sha1Git: | def snapshot_get_random(self, db=None, cur=None) -> Sha1Git: | ||||
return db.snapshot_get_random(cur) | return db.snapshot_get_random(cur) | ||||
▲ Show 20 Lines • Show All 615 Lines • Show Last 20 Lines |