Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/storage.py
| Show First 20 Lines • Show All 959 Lines • ▼ Show 20 Lines | def stat_counters(self): | ||||
| ) | ) | ||||
| stats = {key: 0 for key in keys} | stats = {key: 0 for key in keys} | ||||
| stats.update({row.object_type: row.count for row in rows}) | stats.update({row.object_type: row.count for row in rows}) | ||||
| return stats | return stats | ||||
| def refresh_stat_counters(self): | def refresh_stat_counters(self): | ||||
| pass | pass | ||||
| def object_metadata_add(self, metadata: Iterable[RawExtrinsicMetadata]) -> None: | def raw_extrinsic_metadata_add( | ||||
| self, metadata: Iterable[RawExtrinsicMetadata] | |||||
| ) -> None: | |||||
| for metadata_entry in metadata: | for metadata_entry in metadata: | ||||
| if not self._cql_runner.metadata_authority_get( | if not self._cql_runner.metadata_authority_get( | ||||
| metadata_entry.authority.type.value, metadata_entry.authority.url | metadata_entry.authority.type.value, metadata_entry.authority.url | ||||
| ): | ): | ||||
| raise StorageArgumentException( | raise StorageArgumentException( | ||||
| f"Unknown authority {metadata_entry.authority}" | f"Unknown authority {metadata_entry.authority}" | ||||
| ) | ) | ||||
| if not self._cql_runner.metadata_fetcher_get( | if not self._cql_runner.metadata_fetcher_get( | ||||
| metadata_entry.fetcher.name, metadata_entry.fetcher.version | metadata_entry.fetcher.name, metadata_entry.fetcher.version | ||||
| ): | ): | ||||
| raise StorageArgumentException( | raise StorageArgumentException( | ||||
| f"Unknown fetcher {metadata_entry.fetcher}" | f"Unknown fetcher {metadata_entry.fetcher}" | ||||
| ) | ) | ||||
| try: | try: | ||||
| self._cql_runner.object_metadata_add( | self._cql_runner.raw_extrinsic_metadata_add( | ||||
| type=metadata_entry.type.value, | type=metadata_entry.type.value, | ||||
| id=str(metadata_entry.id), | id=str(metadata_entry.id), | ||||
| authority_type=metadata_entry.authority.type.value, | authority_type=metadata_entry.authority.type.value, | ||||
| authority_url=metadata_entry.authority.url, | authority_url=metadata_entry.authority.url, | ||||
| discovery_date=metadata_entry.discovery_date, | discovery_date=metadata_entry.discovery_date, | ||||
| fetcher_name=metadata_entry.fetcher.name, | fetcher_name=metadata_entry.fetcher.name, | ||||
| fetcher_version=metadata_entry.fetcher.version, | fetcher_version=metadata_entry.fetcher.version, | ||||
| format=metadata_entry.format, | format=metadata_entry.format, | ||||
| metadata=metadata_entry.metadata, | metadata=metadata_entry.metadata, | ||||
| origin=metadata_entry.origin, | origin=metadata_entry.origin, | ||||
| visit=metadata_entry.visit, | visit=metadata_entry.visit, | ||||
| snapshot=map_optional(str, metadata_entry.snapshot), | snapshot=map_optional(str, metadata_entry.snapshot), | ||||
| release=map_optional(str, metadata_entry.release), | release=map_optional(str, metadata_entry.release), | ||||
| revision=map_optional(str, metadata_entry.revision), | revision=map_optional(str, metadata_entry.revision), | ||||
| path=metadata_entry.path, | path=metadata_entry.path, | ||||
| directory=map_optional(str, metadata_entry.directory), | directory=map_optional(str, metadata_entry.directory), | ||||
| ) | ) | ||||
| except TypeError as e: | except TypeError as e: | ||||
| raise StorageArgumentException(*e.args) | raise StorageArgumentException(*e.args) | ||||
| def object_metadata_get( | def raw_extrinsic_metadata_get( | ||||
| self, | self, | ||||
| object_type: MetadataTargetType, | object_type: MetadataTargetType, | ||||
| id: Union[str, SWHID], | id: Union[str, SWHID], | ||||
| authority: MetadataAuthority, | authority: MetadataAuthority, | ||||
| after: Optional[datetime.datetime] = None, | after: Optional[datetime.datetime] = None, | ||||
| page_token: Optional[bytes] = None, | page_token: Optional[bytes] = None, | ||||
| limit: int = 1000, | limit: int = 1000, | ||||
| ) -> Dict[str, Union[Optional[bytes], List[RawExtrinsicMetadata]]]: | ) -> Dict[str, Union[Optional[bytes], List[RawExtrinsicMetadata]]]: | ||||
| if object_type == MetadataTargetType.ORIGIN: | if object_type == MetadataTargetType.ORIGIN: | ||||
| if isinstance(id, SWHID): | if isinstance(id, SWHID): | ||||
| raise StorageArgumentException( | raise StorageArgumentException( | ||||
| f"object_metadata_get called with object_type='origin', but " | f"raw_extrinsic_metadata_get called with object_type='origin', " | ||||
| f"provided id is an SWHID: {id!r}" | f"but provided id is an SWHID: {id!r}" | ||||
| ) | ) | ||||
| else: | else: | ||||
| if not isinstance(id, SWHID): | if not isinstance(id, SWHID): | ||||
| raise StorageArgumentException( | raise StorageArgumentException( | ||||
| f"object_metadata_get called with object_type!='origin', but " | f"raw_extrinsic_metadata_get called with object_type!='origin', " | ||||
| f"provided id is not an SWHID: {id!r}" | f"but provided id is not an SWHID: {id!r}" | ||||
| ) | ) | ||||
| if page_token is not None: | if page_token is not None: | ||||
| (after_date, after_fetcher_name, after_fetcher_url) = msgpack_loads( | (after_date, after_fetcher_name, after_fetcher_url) = msgpack_loads( | ||||
| page_token | page_token | ||||
| ) | ) | ||||
| if after and after_date < after: | if after and after_date < after: | ||||
| raise StorageArgumentException( | raise StorageArgumentException( | ||||
| "page_token is inconsistent with the value of 'after'." | "page_token is inconsistent with the value of 'after'." | ||||
| ) | ) | ||||
| entries = self._cql_runner.object_metadata_get_after_date_and_fetcher( | entries = self._cql_runner.raw_extrinsic_metadata_get_after_date_and_fetcher( # noqa | ||||
| str(id), | str(id), | ||||
| authority.type.value, | authority.type.value, | ||||
| authority.url, | authority.url, | ||||
| after_date, | after_date, | ||||
| after_fetcher_name, | after_fetcher_name, | ||||
| after_fetcher_url, | after_fetcher_url, | ||||
| ) | ) | ||||
| elif after is not None: | elif after is not None: | ||||
| entries = self._cql_runner.object_metadata_get_after_date( | entries = self._cql_runner.raw_extrinsic_metadata_get_after_date( | ||||
| str(id), authority.type.value, authority.url, after | str(id), authority.type.value, authority.url, after | ||||
| ) | ) | ||||
| else: | else: | ||||
| entries = self._cql_runner.object_metadata_get( | entries = self._cql_runner.raw_extrinsic_metadata_get( | ||||
| str(id), authority.type.value, authority.url | str(id), authority.type.value, authority.url | ||||
| ) | ) | ||||
| if limit: | if limit: | ||||
| entries = itertools.islice(entries, 0, limit + 1) | entries = itertools.islice(entries, 0, limit + 1) | ||||
| results = [] | results = [] | ||||
| for entry in entries: | for entry in entries: | ||||
| ▲ Show 20 Lines • Show All 97 Lines • Show Last 20 Lines | |||||