Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/storage.py
Show All 20 Lines | from typing import ( | ||||
Tuple, | Tuple, | ||||
Union, | Union, | ||||
) | ) | ||||
import attr | import attr | ||||
from swh.core.api.serializers import msgpack_dumps, msgpack_loads | from swh.core.api.serializers import msgpack_dumps, msgpack_loads | ||||
from swh.model.hashutil import DEFAULT_ALGORITHMS | from swh.model.hashutil import DEFAULT_ALGORITHMS | ||||
from swh.model.identifiers import SWHID, parse_swhid | |||||
from swh.model.model import ( | from swh.model.model import ( | ||||
Content, | Content, | ||||
Directory, | Directory, | ||||
DirectoryEntry, | DirectoryEntry, | ||||
MetadataAuthority, | MetadataAuthority, | ||||
MetadataAuthorityType, | MetadataAuthorityType, | ||||
MetadataFetcher, | MetadataFetcher, | ||||
MetadataTargetType, | |||||
Origin, | Origin, | ||||
OriginVisit, | OriginVisit, | ||||
OriginVisitStatus, | OriginVisitStatus, | ||||
RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
Release, | Release, | ||||
Revision, | Revision, | ||||
Sha1Git, | Sha1Git, | ||||
SkippedContent, | SkippedContent, | ||||
Snapshot, | Snapshot, | ||||
SnapshotBranch, | SnapshotBranch, | ||||
TargetType, | TargetType, | ||||
) | ) | ||||
from swh.model.swhid import SWHID, parse_swhid | |||||
from swh.storage.interface import ( | from swh.storage.interface import ( | ||||
VISIT_STATUSES, | VISIT_STATUSES, | ||||
ListOrder, | ListOrder, | ||||
PagedResult, | PagedResult, | ||||
PartialBranches, | PartialBranches, | ||||
Sha1, | Sha1, | ||||
) | ) | ||||
from swh.storage.objstorage import ObjStorage | from swh.storage.objstorage import ObjStorage | ||||
▲ Show 20 Lines • Show All 1,094 Lines • ▼ Show 20 Lines | def raw_extrinsic_metadata_add(self, metadata: List[RawExtrinsicMetadata]) -> None: | ||||
metadata_entry.fetcher.name, metadata_entry.fetcher.version | metadata_entry.fetcher.name, metadata_entry.fetcher.version | ||||
): | ): | ||||
raise StorageArgumentException( | raise StorageArgumentException( | ||||
f"Unknown fetcher {metadata_entry.fetcher}" | f"Unknown fetcher {metadata_entry.fetcher}" | ||||
) | ) | ||||
try: | try: | ||||
row = RawExtrinsicMetadataRow( | row = RawExtrinsicMetadataRow( | ||||
type=metadata_entry.type.value, | |||||
target=str(metadata_entry.target), | target=str(metadata_entry.target), | ||||
authority_type=metadata_entry.authority.type.value, | authority_type=metadata_entry.authority.type.value, | ||||
authority_url=metadata_entry.authority.url, | authority_url=metadata_entry.authority.url, | ||||
discovery_date=metadata_entry.discovery_date, | discovery_date=metadata_entry.discovery_date, | ||||
fetcher_name=metadata_entry.fetcher.name, | fetcher_name=metadata_entry.fetcher.name, | ||||
fetcher_version=metadata_entry.fetcher.version, | fetcher_version=metadata_entry.fetcher.version, | ||||
format=metadata_entry.format, | format=metadata_entry.format, | ||||
metadata=metadata_entry.metadata, | metadata=metadata_entry.metadata, | ||||
origin=metadata_entry.origin, | origin=map_optional(str, metadata_entry.origin), | ||||
visit=metadata_entry.visit, | visit=metadata_entry.visit, | ||||
snapshot=map_optional(str, metadata_entry.snapshot), | snapshot=map_optional(str, metadata_entry.snapshot), | ||||
release=map_optional(str, metadata_entry.release), | release=map_optional(str, metadata_entry.release), | ||||
revision=map_optional(str, metadata_entry.revision), | revision=map_optional(str, metadata_entry.revision), | ||||
path=metadata_entry.path, | path=metadata_entry.path, | ||||
directory=map_optional(str, metadata_entry.directory), | directory=map_optional(str, metadata_entry.directory), | ||||
) | ) | ||||
self._cql_runner.raw_extrinsic_metadata_add(row) | self._cql_runner.raw_extrinsic_metadata_add(row) | ||||
except TypeError as e: | except TypeError as e: | ||||
raise StorageArgumentException(*e.args) | raise StorageArgumentException(*e.args) | ||||
def raw_extrinsic_metadata_get( | def raw_extrinsic_metadata_get( | ||||
self, | self, | ||||
type: MetadataTargetType, | target: SWHID, | ||||
target: Union[str, SWHID], | |||||
authority: MetadataAuthority, | authority: MetadataAuthority, | ||||
after: Optional[datetime.datetime] = None, | after: Optional[datetime.datetime] = None, | ||||
page_token: Optional[bytes] = None, | page_token: Optional[bytes] = None, | ||||
limit: int = 1000, | limit: int = 1000, | ||||
) -> PagedResult[RawExtrinsicMetadata]: | ) -> PagedResult[RawExtrinsicMetadata]: | ||||
if type == MetadataTargetType.ORIGIN: | |||||
if isinstance(target, SWHID): | |||||
raise StorageArgumentException( | |||||
f"raw_extrinsic_metadata_get called with type='origin', " | |||||
f"but provided target is a SWHID: {target!r}" | |||||
) | |||||
else: | |||||
if not isinstance(target, SWHID): | |||||
raise StorageArgumentException( | |||||
f"raw_extrinsic_metadata_get called with type!='origin', " | |||||
f"but provided target is not a SWHID: {target!r}" | |||||
) | |||||
if page_token is not None: | if page_token is not None: | ||||
(after_date, after_fetcher_name, after_fetcher_url) = msgpack_loads( | (after_date, after_fetcher_name, after_fetcher_url) = msgpack_loads( | ||||
base64.b64decode(page_token) | base64.b64decode(page_token) | ||||
) | ) | ||||
if after and after_date < after: | if after and after_date < after: | ||||
raise StorageArgumentException( | raise StorageArgumentException( | ||||
"page_token is inconsistent with the value of 'after'." | "page_token is inconsistent with the value of 'after'." | ||||
) | ) | ||||
Show All 19 Lines | ) -> PagedResult[RawExtrinsicMetadata]: | ||||
results = [] | results = [] | ||||
for entry in entries: | for entry in entries: | ||||
discovery_date = entry.discovery_date.replace(tzinfo=datetime.timezone.utc) | discovery_date = entry.discovery_date.replace(tzinfo=datetime.timezone.utc) | ||||
assert str(target) == entry.target | assert str(target) == entry.target | ||||
result = RawExtrinsicMetadata( | result = RawExtrinsicMetadata( | ||||
type=MetadataTargetType(entry.type), | |||||
target=target, | target=target, | ||||
authority=MetadataAuthority( | authority=MetadataAuthority( | ||||
type=MetadataAuthorityType(entry.authority_type), | type=MetadataAuthorityType(entry.authority_type), | ||||
url=entry.authority_url, | url=entry.authority_url, | ||||
), | ), | ||||
fetcher=MetadataFetcher( | fetcher=MetadataFetcher( | ||||
name=entry.fetcher_name, version=entry.fetcher_version, | name=entry.fetcher_name, version=entry.fetcher_version, | ||||
), | ), | ||||
discovery_date=discovery_date, | discovery_date=discovery_date, | ||||
format=entry.format, | format=entry.format, | ||||
metadata=entry.metadata, | metadata=entry.metadata, | ||||
origin=entry.origin, | origin=map_optional(parse_swhid, entry.origin), | ||||
visit=entry.visit, | visit=entry.visit, | ||||
snapshot=map_optional(parse_swhid, entry.snapshot), | snapshot=map_optional(parse_swhid, entry.snapshot), | ||||
release=map_optional(parse_swhid, entry.release), | release=map_optional(parse_swhid, entry.release), | ||||
revision=map_optional(parse_swhid, entry.revision), | revision=map_optional(parse_swhid, entry.revision), | ||||
path=entry.path, | path=entry.path, | ||||
directory=map_optional(parse_swhid, entry.directory), | directory=map_optional(parse_swhid, entry.directory), | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 76 Lines • Show Last 20 Lines |