Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/storage.py
Show First 20 Lines • Show All 918 Lines • ▼ Show 20 Lines | ): | ||||
if snapshot and snapshot != visit["snapshot"]: | if snapshot and snapshot != visit["snapshot"]: | ||||
updates["snapshot"] = snapshot | updates["snapshot"] = snapshot | ||||
if updates: | if updates: | ||||
with convert_validation_exceptions(): | with convert_validation_exceptions(): | ||||
updated_visit = OriginVisit.from_dict({**visit, **updates}) | updated_visit = OriginVisit.from_dict({**visit, **updates}) | ||||
self.journal_writer.origin_visit_update(updated_visit) | self.journal_writer.origin_visit_update(updated_visit) | ||||
# Write updates to origin visit (backward compatibility) | |||||
db.origin_visit_update(origin, visit_id, updates) | |||||
# Add new origin visit status | |||||
last_visit_status = self._origin_visit_get_updated( | last_visit_status = self._origin_visit_get_updated( | ||||
origin, visit_id, db=db, cur=cur | origin, visit_id, db=db, cur=cur | ||||
) | ) | ||||
assert last_visit_status is not None | assert last_visit_status is not None | ||||
with convert_validation_exceptions(): | with convert_validation_exceptions(): | ||||
visit_status = OriginVisitStatus( | visit_status = OriginVisitStatus( | ||||
origin=origin_url, | origin=origin_url, | ||||
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines | class Storage: | ||||
def origin_visit_get( | def origin_visit_get( | ||||
self, | self, | ||||
origin: str, | origin: str, | ||||
last_visit: Optional[int] = None, | last_visit: Optional[int] = None, | ||||
limit: Optional[int] = None, | limit: Optional[int] = None, | ||||
db=None, | db=None, | ||||
cur=None, | cur=None, | ||||
) -> Iterable[Dict[str, Any]]: | ) -> Iterable[Dict[str, Any]]: | ||||
lines = db.origin_visit_get_all( | for line in db.origin_visit_get_all( | ||||
origin, last_visit=last_visit, limit=limit, cur=cur | origin, last_visit=last_visit, limit=limit, cur=cur | ||||
) | ): | ||||
for line in lines: | data = dict(zip(db.origin_visit_get_cols, line)) | ||||
visit = dict(zip(db.origin_visit_get_cols, line)) | yield data | ||||
yield self._origin_visit_apply_update(visit, db) | |||||
@timed | @timed | ||||
@db_transaction(statement_timeout=500) | @db_transaction(statement_timeout=500) | ||||
def origin_visit_find_by_date( | def origin_visit_find_by_date( | ||||
self, origin: str, visit_date: datetime.datetime, db=None, cur=None | self, origin: str, visit_date: datetime.datetime, db=None, cur=None | ||||
) -> Optional[Dict[str, Any]]: | ) -> Optional[Dict[str, Any]]: | ||||
visit = db.origin_visit_find_by_date(origin, visit_date, cur=cur) | visit = db.origin_visit_find_by_date(origin, visit_date, cur=cur) | ||||
if visit: | if visit: | ||||
return self._origin_visit_apply_update(visit, db) | return visit | ||||
return None | return None | ||||
@timed | @timed | ||||
@db_transaction(statement_timeout=500) | @db_transaction(statement_timeout=500) | ||||
def origin_visit_get_by( | def origin_visit_get_by( | ||||
self, origin: str, visit: int, db=None, cur=None | self, origin: str, visit: int, db=None, cur=None | ||||
) -> Optional[Dict[str, Any]]: | ) -> Optional[Dict[str, Any]]: | ||||
row = db.origin_visit_get(origin, visit, cur) | ori_visit = db.origin_visit_get(origin, visit, cur) | ||||
if row: | if not ori_visit: | ||||
visit_dict = dict(zip(db.origin_visit_get_cols, row)) | |||||
return self._origin_visit_apply_update(visit_dict, db) | |||||
return None | return None | ||||
return dict(zip(db.origin_visit_get_cols, ori_visit)) | |||||
@timed | @timed | ||||
@db_transaction(statement_timeout=4000) | @db_transaction(statement_timeout=4000) | ||||
def origin_visit_get_latest( | def origin_visit_get_latest( | ||||
self, | self, | ||||
origin: str, | origin: str, | ||||
allowed_statuses: Optional[List[str]] = None, | allowed_statuses: Optional[List[str]] = None, | ||||
require_snapshot: bool = False, | require_snapshot: bool = False, | ||||
db=None, | db=None, | ||||
cur=None, | cur=None, | ||||
) -> Optional[Dict[str, Any]]: | ) -> Optional[Dict[str, Any]]: | ||||
row = db.origin_visit_get_latest( | origin_visit = db.origin_visit_get_latest( | ||||
origin, | origin, | ||||
allowed_statuses=allowed_statuses, | allowed_statuses=allowed_statuses, | ||||
require_snapshot=require_snapshot, | require_snapshot=require_snapshot, | ||||
cur=cur, | cur=cur, | ||||
) | ) | ||||
if row: | if origin_visit: | ||||
visit = dict(zip(db.origin_visit_get_cols, row)) | return dict(zip(db.origin_visit_get_cols, origin_visit)) | ||||
return self._origin_visit_apply_update(visit, db) | |||||
return None | return None | ||||
@timed | @timed | ||||
@db_transaction() | @db_transaction() | ||||
def origin_visit_get_random( | def origin_visit_get_random( | ||||
self, type: str, db=None, cur=None | self, type: str, db=None, cur=None | ||||
) -> Optional[Dict[str, Any]]: | ) -> Optional[Dict[str, Any]]: | ||||
row = db.origin_visit_get_random(type, cur) | result = db.origin_visit_get_random(type, cur) | ||||
if row: | if result: | ||||
visit = dict(zip(db.origin_visit_get_cols, row)) | return dict(zip(db.origin_visit_get_cols, result)) | ||||
return self._origin_visit_apply_update(visit, db) | else: | ||||
return None | return None | ||||
@timed | @timed | ||||
@db_transaction(statement_timeout=2000) | @db_transaction(statement_timeout=2000) | ||||
def object_find_by_sha1_git(self, ids, db=None, cur=None): | def object_find_by_sha1_git(self, ids, db=None, cur=None): | ||||
ret = {id: [] for id in ids} | ret = {id: [] for id in ids} | ||||
for retval in db.object_find_by_sha1_git(ids, cur=cur): | for retval in db.object_find_by_sha1_git(ids, cur=cur): | ||||
if retval[1]: | if retval[1]: | ||||
▲ Show 20 Lines • Show All 233 Lines • Show Last 20 Lines |