Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/storage.py
| Show First 20 Lines • Show All 1,190 Lines • ▼ Show 20 Lines | def raw_extrinsic_metadata_add(self, metadata: List[RawExtrinsicMetadata]) -> None: | ||||
| metadata_entry.fetcher.name, metadata_entry.fetcher.version | metadata_entry.fetcher.name, metadata_entry.fetcher.version | ||||
| ): | ): | ||||
| raise StorageArgumentException( | raise StorageArgumentException( | ||||
| f"Unknown fetcher {metadata_entry.fetcher}" | f"Unknown fetcher {metadata_entry.fetcher}" | ||||
| ) | ) | ||||
| try: | try: | ||||
| row = RawExtrinsicMetadataRow( | row = RawExtrinsicMetadataRow( | ||||
| id=metadata_entry.id, | |||||
| type=metadata_entry.target.object_type.name.lower(), | type=metadata_entry.target.object_type.name.lower(), | ||||
| target=str(metadata_entry.target), | target=str(metadata_entry.target), | ||||
| authority_type=metadata_entry.authority.type.value, | authority_type=metadata_entry.authority.type.value, | ||||
| authority_url=metadata_entry.authority.url, | authority_url=metadata_entry.authority.url, | ||||
| discovery_date=metadata_entry.discovery_date, | discovery_date=metadata_entry.discovery_date, | ||||
| fetcher_name=metadata_entry.fetcher.name, | fetcher_name=metadata_entry.fetcher.name, | ||||
| fetcher_version=metadata_entry.fetcher.version, | fetcher_version=metadata_entry.fetcher.version, | ||||
| format=metadata_entry.format, | format=metadata_entry.format, | ||||
| Show All 14 Lines | def raw_extrinsic_metadata_get( | ||||
| self, | self, | ||||
| target: ExtendedSWHID, | target: ExtendedSWHID, | ||||
| authority: MetadataAuthority, | authority: MetadataAuthority, | ||||
| after: Optional[datetime.datetime] = None, | after: Optional[datetime.datetime] = None, | ||||
| page_token: Optional[bytes] = None, | page_token: Optional[bytes] = None, | ||||
| limit: int = 1000, | limit: int = 1000, | ||||
| ) -> PagedResult[RawExtrinsicMetadata]: | ) -> PagedResult[RawExtrinsicMetadata]: | ||||
| if page_token is not None: | if page_token is not None: | ||||
| (after_date, after_fetcher_name, after_fetcher_url) = msgpack_loads( | (after_date, id_) = msgpack_loads(base64.b64decode(page_token)) | ||||
| base64.b64decode(page_token) | |||||
| ) | |||||
| if after and after_date < after: | if after and after_date < after: | ||||
| raise StorageArgumentException( | raise StorageArgumentException( | ||||
| "page_token is inconsistent with the value of 'after'." | "page_token is inconsistent with the value of 'after'." | ||||
| ) | ) | ||||
| entries = self._cql_runner.raw_extrinsic_metadata_get_after_date_and_fetcher( # noqa | entries = self._cql_runner.raw_extrinsic_metadata_get_after_date_and_id( | ||||
| str(target), | str(target), authority.type.value, authority.url, after_date, id_, | ||||
| authority.type.value, | |||||
| authority.url, | |||||
| after_date, | |||||
| after_fetcher_name, | |||||
| after_fetcher_url, | |||||
| ) | ) | ||||
| elif after is not None: | elif after is not None: | ||||
| entries = self._cql_runner.raw_extrinsic_metadata_get_after_date( | entries = self._cql_runner.raw_extrinsic_metadata_get_after_date( | ||||
| str(target), authority.type.value, authority.url, after | str(target), authority.type.value, authority.url, after | ||||
| ) | ) | ||||
| else: | else: | ||||
| entries = self._cql_runner.raw_extrinsic_metadata_get( | entries = self._cql_runner.raw_extrinsic_metadata_get( | ||||
| str(target), authority.type.value, authority.url | str(target), authority.type.value, authority.url | ||||
| Show All 31 Lines | ) -> PagedResult[RawExtrinsicMetadata]: | ||||
| results.append(result) | results.append(result) | ||||
| if len(results) > limit: | if len(results) > limit: | ||||
| results.pop() | results.pop() | ||||
| assert len(results) == limit | assert len(results) == limit | ||||
| last_result = results[-1] | last_result = results[-1] | ||||
| next_page_token: Optional[str] = base64.b64encode( | next_page_token: Optional[str] = base64.b64encode( | ||||
| msgpack_dumps( | msgpack_dumps((last_result.discovery_date, last_result.id,)) | ||||
| ( | |||||
| last_result.discovery_date, | |||||
| last_result.fetcher.name, | |||||
| last_result.fetcher.version, | |||||
| ) | |||||
| ) | |||||
| ).decode() | ).decode() | ||||
| else: | else: | ||||
| next_page_token = None | next_page_token = None | ||||
| return PagedResult(next_page_token=next_page_token, results=results,) | return PagedResult(next_page_token=next_page_token, results=results,) | ||||
| def metadata_fetcher_add(self, fetchers: List[MetadataFetcher]) -> None: | def metadata_fetcher_add(self, fetchers: List[MetadataFetcher]) -> None: | ||||
| self.journal_writer.metadata_fetcher_add(fetchers) | self.journal_writer.metadata_fetcher_add(fetchers) | ||||
| ▲ Show 20 Lines • Show All 135 Lines • Show Last 20 Lines | |||||