Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/storage.py
| Show All 26 Lines | from swh.model.model import ( | ||||
| Snapshot, | Snapshot, | ||||
| Origin, | Origin, | ||||
| MetadataAuthority, | MetadataAuthority, | ||||
| MetadataAuthorityType, | MetadataAuthorityType, | ||||
| MetadataFetcher, | MetadataFetcher, | ||||
| MetadataTargetType, | MetadataTargetType, | ||||
| RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
| ) | ) | ||||
| from swh.storage.interface import PagedResult | |||||
| from swh.storage.objstorage import ObjStorage | from swh.storage.objstorage import ObjStorage | ||||
| from swh.storage.writer import JournalWriter | from swh.storage.writer import JournalWriter | ||||
| from swh.storage.utils import map_optional, now | from swh.storage.utils import map_optional, now | ||||
| from ..exc import StorageArgumentException, HashCollision | from ..exc import StorageArgumentException, HashCollision | ||||
| from .common import TOKEN_BEGIN, TOKEN_END | from .common import TOKEN_BEGIN, TOKEN_END | ||||
| from . import converters | from . import converters | ||||
| from .cql import CqlRunner | from .cql import CqlRunner | ||||
| ▲ Show 20 Lines • Show All 796 Lines • ▼ Show 20 Lines | def _format_origin_visit_row(visit): | ||||
| **visit._asdict(), | **visit._asdict(), | ||||
| "origin": visit.origin, | "origin": visit.origin, | ||||
| "date": visit.date.replace(tzinfo=datetime.timezone.utc), | "date": visit.date.replace(tzinfo=datetime.timezone.utc), | ||||
| } | } | ||||
| def origin_visit_get( | def origin_visit_get( | ||||
| self, | self, | ||||
| origin: str, | origin: str, | ||||
| last_visit: Optional[int] = None, | page_token: Optional[str] = None, | ||||
| limit: Optional[int] = None, | |||||
| order: str = "asc", | order: str = "asc", | ||||
| ) -> Iterable[Dict[str, Any]]: | limit: int = 10, | ||||
| rows = self._cql_runner.origin_visit_get(origin, last_visit, limit, order) | ) -> PagedResult[OriginVisit]: | ||||
| order = order.lower() | |||||
| allowed_orders = ["asc", "desc"] | |||||
| if order not in allowed_orders: | |||||
| raise StorageArgumentException( | |||||
| f"order must be one of {', '.join(allowed_orders)}." | |||||
| ) | |||||
vlorentz: out of scope, but cool | |||||
Done Inline Actionswell, we rewrote entirely the endpoints, might as well unify the storage argument exception along the way. ardumont: well, we rewrote entirely the endpoints, might as well unify the storage argument exception… | |||||
| if page_token and not isinstance(page_token, str): | |||||
| raise StorageArgumentException("page_token must be a string.") | |||||
| next_page_token = None | |||||
| visit_from = page_token and int(page_token) | |||||
| visits: List[OriginVisit] = [] | |||||
| extra_limit = limit + 1 | |||||
| rows = self._cql_runner.origin_visit_get(origin, visit_from, extra_limit, order) | |||||
| for row in rows: | for row in rows: | ||||
| visit = self._format_origin_visit_row(row) | visits.append(converters.row_to_visit(row)) | ||||
| yield self._origin_visit_apply_last_status(visit) | |||||
| assert len(visits) <= extra_limit | |||||
| if len(visits) == extra_limit: | |||||
| last_visit = visits[limit] | |||||
| visits = visits[:limit] | |||||
| assert last_visit is not None | |||||
| next_page_token = str(last_visit.visit) | |||||
| return PagedResult(results=visits, next_page_token=next_page_token) | |||||
| def origin_visit_find_by_date( | def origin_visit_find_by_date( | ||||
| self, origin: str, visit_date: datetime.datetime | self, origin: str, visit_date: datetime.datetime | ||||
| ) -> Optional[OriginVisit]: | ) -> Optional[OriginVisit]: | ||||
| # Iterator over all the visits of the origin | # Iterator over all the visits of the origin | ||||
| # This should be ok for now, as there aren't too many visits | # This should be ok for now, as there aren't too many visits | ||||
| # per origin. | # per origin. | ||||
| rows = list(self._cql_runner.origin_visit_get_all(origin)) | rows = list(self._cql_runner.origin_visit_get_all(origin)) | ||||
| ▲ Show 20 Lines • Show All 296 Lines • Show Last 20 Lines | |||||
out of scope, but cool