Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/storage.py
| Show All 27 Lines | from swh.model.model import ( | ||||
| Snapshot, | Snapshot, | ||||
| Origin, | Origin, | ||||
| MetadataAuthority, | MetadataAuthority, | ||||
| MetadataAuthorityType, | MetadataAuthorityType, | ||||
| MetadataFetcher, | MetadataFetcher, | ||||
| MetadataTargetType, | MetadataTargetType, | ||||
| RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
| ) | ) | ||||
| from swh.storage.interface import ListOrder, PagedResult | from swh.storage.interface import ListOrder, PagedResult, VISIT_STATUSES | ||||
| from swh.storage.objstorage import ObjStorage | from swh.storage.objstorage import ObjStorage | ||||
| from swh.storage.writer import JournalWriter | from swh.storage.writer import JournalWriter | ||||
| from swh.storage.utils import map_optional, now | from swh.storage.utils import map_optional, now | ||||
| from ..exc import StorageArgumentException, HashCollision | from ..exc import StorageArgumentException, HashCollision | ||||
| from .common import TOKEN_BEGIN, TOKEN_END | from .common import TOKEN_BEGIN, TOKEN_END | ||||
| from . import converters | from . import converters | ||||
| from .cql import CqlRunner | from .cql import CqlRunner | ||||
| ▲ Show 20 Lines • Show All 891 Lines • ▼ Show 20 Lines | class CassandraStorage: | ||||
| def origin_visit_get_latest( | def origin_visit_get_latest( | ||||
| self, | self, | ||||
| origin: str, | origin: str, | ||||
| type: Optional[str] = None, | type: Optional[str] = None, | ||||
| allowed_statuses: Optional[List[str]] = None, | allowed_statuses: Optional[List[str]] = None, | ||||
| require_snapshot: bool = False, | require_snapshot: bool = False, | ||||
| ) -> Optional[OriginVisit]: | ) -> Optional[OriginVisit]: | ||||
| if allowed_statuses and not set(allowed_statuses).intersection(VISIT_STATUSES): | |||||
| raise StorageArgumentException( | |||||
| f"Unknown allowed statuses {','.join(allowed_statuses)}, only " | |||||
| f"{','.join(VISIT_STATUSES)} authorized" | |||||
| ) | |||||
| # TODO: Do not fetch all visits | # TODO: Do not fetch all visits | ||||
| rows = self._cql_runner.origin_visit_get_all(origin) | rows = self._cql_runner.origin_visit_get_all(origin) | ||||
| latest_visit = None | latest_visit = None | ||||
| for row in rows: | for row in rows: | ||||
| visit = self._format_origin_visit_row(row) | visit = self._format_origin_visit_row(row) | ||||
| updated_visit = self._origin_visit_apply_last_status(visit) | updated_visit = self._origin_visit_apply_last_status(visit) | ||||
| if type is not None and updated_visit["type"] != type: | if type is not None and updated_visit["type"] != type: | ||||
| continue | continue | ||||
| Show All 22 Lines | class CassandraStorage: | ||||
| def origin_visit_status_get_latest( | def origin_visit_status_get_latest( | ||||
| self, | self, | ||||
| origin_url: str, | origin_url: str, | ||||
| visit: int, | visit: int, | ||||
| allowed_statuses: Optional[List[str]] = None, | allowed_statuses: Optional[List[str]] = None, | ||||
| require_snapshot: bool = False, | require_snapshot: bool = False, | ||||
| ) -> Optional[OriginVisitStatus]: | ) -> Optional[OriginVisitStatus]: | ||||
| if allowed_statuses and not set(allowed_statuses).intersection(VISIT_STATUSES): | |||||
| raise StorageArgumentException( | |||||
| f"Unknown allowed statuses {','.join(allowed_statuses)}, only " | |||||
| f"{','.join(VISIT_STATUSES)} authorized" | |||||
| ) | |||||
| rows = self._cql_runner.origin_visit_status_get( | rows = self._cql_runner.origin_visit_status_get( | ||||
| origin_url, visit, allowed_statuses, require_snapshot | origin_url, visit, allowed_statuses, require_snapshot | ||||
| ) | ) | ||||
| # filtering is done python side as we cannot do it server side | # filtering is done python side as we cannot do it server side | ||||
| if allowed_statuses: | if allowed_statuses: | ||||
| rows = [row for row in rows if row.status in allowed_statuses] | rows = [row for row in rows if row.status in allowed_statuses] | ||||
| if require_snapshot: | if require_snapshot: | ||||
| rows = [row for row in rows if row.snapshot is not None] | rows = [row for row in rows if row.snapshot is not None] | ||||
| ▲ Show 20 Lines • Show All 228 Lines • Show Last 20 Lines | |||||