Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/storage.py
Show All 39 Lines | from swh.model.model import ( | ||||
SHA1_SIZE, | SHA1_SIZE, | ||||
MetadataAuthority, | MetadataAuthority, | ||||
MetadataAuthorityType, | MetadataAuthorityType, | ||||
MetadataFetcher, | MetadataFetcher, | ||||
MetadataTargetType, | MetadataTargetType, | ||||
RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
) | ) | ||||
from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex | from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex | ||||
from swh.storage.interface import PagedResult | |||||
from swh.storage.objstorage import ObjStorage | from swh.storage.objstorage import ObjStorage | ||||
from swh.storage.utils import now | from swh.storage.utils import now | ||||
from . import converters | from . import converters | ||||
from .common import db_transaction_generator, db_transaction | from .common import db_transaction_generator, db_transaction | ||||
from .db import Db | from .db import Db | ||||
from .exc import StorageArgumentException, StorageDBError, HashCollision | from .exc import StorageArgumentException, StorageDBError, HashCollision | ||||
from .algos import diff | from .algos import diff | ||||
▲ Show 20 Lines • Show All 816 Lines • ▼ Show 20 Lines | ) -> Optional[OriginVisitStatus]: | ||||
row = db.origin_visit_status_get_latest( | row = db.origin_visit_status_get_latest( | ||||
origin_url, visit, allowed_statuses, require_snapshot, cur=cur | origin_url, visit, allowed_statuses, require_snapshot, cur=cur | ||||
) | ) | ||||
if not row: | if not row: | ||||
return None | return None | ||||
return OriginVisitStatus.from_dict(row) | return OriginVisitStatus.from_dict(row) | ||||
@timed | @timed | ||||
@db_transaction_generator(statement_timeout=500) | @db_transaction(statement_timeout=500) | ||||
def origin_visit_get( | def origin_visit_get( | ||||
self, | self, | ||||
origin: str, | origin: str, | ||||
last_visit: Optional[int] = None, | page_token: Optional[str] = None, | ||||
limit: Optional[int] = None, | |||||
order: str = "asc", | order: str = "asc", | ||||
limit: int = 10, | |||||
db=None, | db=None, | ||||
cur=None, | cur=None, | ||||
) -> Iterable[Dict[str, Any]]: | ) -> PagedResult[OriginVisit]: | ||||
assert order in ["asc", "desc"] | page_token = page_token or "0" | ||||
lines = db.origin_visit_get_all( | order = order.lower() | ||||
origin, last_visit=last_visit, limit=limit, order=order, cur=cur | allowed_orders = ["asc", "desc"] | ||||
if order not in allowed_orders: | |||||
raise StorageArgumentException( | |||||
f"order must be one of {', '.join(allowed_orders)}." | |||||
) | ) | ||||
for line in lines: | if not isinstance(page_token, str): | ||||
yield dict(zip(db.origin_visit_get_cols, line)) | raise StorageArgumentException("page_token must be a string.") | ||||
next_page_token = None | |||||
visit_from = int(page_token) | |||||
visits: List[OriginVisit] = [] | |||||
extra_limit = limit + 1 | |||||
for row in db.origin_visit_get_range( | |||||
origin, visit_from=visit_from, order=order, limit=extra_limit, cur=cur | |||||
): | |||||
row_d = dict(zip(db.origin_visit_cols, row)) | |||||
visits.append( | |||||
OriginVisit( | |||||
origin=row_d["origin"], | |||||
vlorentz: Why `int()`? | |||||
Done Inline Actionsleftover from early dev where mypy said things i misunderstood. ardumont: leftover from early dev where mypy said things i misunderstood.
I'll remove. | |||||
visit=row_d["visit"], | |||||
date=row_d["date"], | |||||
type=row_d["type"], | |||||
) | |||||
) | |||||
assert len(visits) <= extra_limit | |||||
if len(visits) == extra_limit: | |||||
last_visit = visits[limit] | |||||
visits = visits[:limit] | |||||
assert last_visit is not None and last_visit.visit is not None | |||||
if order == ListOrder.ASC: | |||||
next_page_token = str(last_visit.visit - 1) | |||||
else: | |||||
next_page_token = str(last_visit.visit + 1) | |||||
return PagedResult(results=visits, next_page_token=next_page_token) | |||||
@timed | @timed | ||||
@db_transaction(statement_timeout=500) | @db_transaction(statement_timeout=500) | ||||
def origin_visit_find_by_date( | def origin_visit_find_by_date( | ||||
self, origin: str, visit_date: datetime.datetime, db=None, cur=None | self, origin: str, visit_date: datetime.datetime, db=None, cur=None | ||||
) -> Optional[OriginVisit]: | ) -> Optional[OriginVisit]: | ||||
row_d = db.origin_visit_find_by_date(origin, visit_date, cur=cur) | row_d = db.origin_visit_find_by_date(origin, visit_date, cur=cur) | ||||
if not row_d: | if not row_d: | ||||
▲ Show 20 Lines • Show All 440 Lines • Show Last 20 Lines |
Why int()?