Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/in_memory.py
| # Copyright (C) 2015-2020 The Software Heritage developers | # Copyright (C) 2015-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import re | import base64 | ||||
| import bisect | import bisect | ||||
| import collections | import collections | ||||
| import copy | import copy | ||||
| import datetime | import datetime | ||||
| import itertools | import itertools | ||||
| import random | import random | ||||
| import re | |||||
| from collections import defaultdict | from collections import defaultdict | ||||
| from datetime import timedelta | from datetime import timedelta | ||||
| from typing import ( | from typing import ( | ||||
| Any, | Any, | ||||
| Callable, | Callable, | ||||
| Dict, | Dict, | ||||
| Generic, | Generic, | ||||
| ▲ Show 20 Lines • Show All 1,088 Lines • ▼ Show 20 Lines | class InMemoryStorage: | ||||
| def raw_extrinsic_metadata_get( | def raw_extrinsic_metadata_get( | ||||
| self, | self, | ||||
| object_type: MetadataTargetType, | object_type: MetadataTargetType, | ||||
| id: Union[str, SWHID], | id: Union[str, SWHID], | ||||
| authority: MetadataAuthority, | authority: MetadataAuthority, | ||||
| after: Optional[datetime.datetime] = None, | after: Optional[datetime.datetime] = None, | ||||
| page_token: Optional[bytes] = None, | page_token: Optional[bytes] = None, | ||||
| limit: int = 1000, | limit: int = 1000, | ||||
| ) -> Dict[str, Union[Optional[bytes], List[RawExtrinsicMetadata]]]: | ) -> PagedResult[RawExtrinsicMetadata]: | ||||
| authority_key = self._metadata_authority_key(authority) | authority_key = self._metadata_authority_key(authority) | ||||
| if object_type == MetadataTargetType.ORIGIN: | if object_type == MetadataTargetType.ORIGIN: | ||||
| if isinstance(id, SWHID): | if isinstance(id, SWHID): | ||||
| raise StorageArgumentException( | raise StorageArgumentException( | ||||
| f"raw_extrinsic_metadata_get called with object_type='origin', " | f"raw_extrinsic_metadata_get called with object_type='origin', " | ||||
| f"but provided id is an SWHID: {id!r}" | f"but provided id is an SWHID: {id!r}" | ||||
| ) | ) | ||||
| else: | else: | ||||
| if not isinstance(id, SWHID): | if not isinstance(id, SWHID): | ||||
| raise StorageArgumentException( | raise StorageArgumentException( | ||||
| f"raw_extrinsic_metadata_get called with object_type!='origin', " | f"raw_extrinsic_metadata_get called with object_type!='origin', " | ||||
| f"but provided id is not an SWHID: {id!r}" | f"but provided id is not an SWHID: {id!r}" | ||||
| ) | ) | ||||
| if page_token is not None: | if page_token is not None: | ||||
| (after_time, after_fetcher) = msgpack_loads(page_token) | (after_time, after_fetcher) = msgpack_loads(base64.b64decode(page_token)) | ||||
| after_fetcher = tuple(after_fetcher) | after_fetcher = tuple(after_fetcher) | ||||
| if after is not None and after > after_time: | if after is not None and after > after_time: | ||||
| raise StorageArgumentException( | raise StorageArgumentException( | ||||
| "page_token is inconsistent with the value of 'after'." | "page_token is inconsistent with the value of 'after'." | ||||
| ) | ) | ||||
| entries = self._raw_extrinsic_metadata[object_type][id][ | entries = self._raw_extrinsic_metadata[object_type][id][ | ||||
| authority_key | authority_key | ||||
| ].iter_after((after_time, after_fetcher)) | ].iter_after((after_time, after_fetcher)) | ||||
| Show All 25 Lines | ) -> PagedResult[RawExtrinsicMetadata]: | ||||
| fetcher=attr.evolve(entry_fetcher, metadata=None), | fetcher=attr.evolve(entry_fetcher, metadata=None), | ||||
| ) | ) | ||||
| ) | ) | ||||
| if len(results) > limit: | if len(results) > limit: | ||||
| results.pop() | results.pop() | ||||
| assert len(results) == limit | assert len(results) == limit | ||||
| last_result = results[-1] | last_result = results[-1] | ||||
| next_page_token: Optional[bytes] = msgpack_dumps( | next_page_token: Optional[str] = base64.b64encode( | ||||
| msgpack_dumps( | |||||
| ( | ( | ||||
| last_result.discovery_date, | last_result.discovery_date, | ||||
| self._metadata_fetcher_key(last_result.fetcher), | self._metadata_fetcher_key(last_result.fetcher), | ||||
| ) | ) | ||||
| ) | ) | ||||
| ).decode() | |||||
| else: | else: | ||||
| next_page_token = None | next_page_token = None | ||||
| return { | return PagedResult(next_page_token=next_page_token, results=results,) | ||||
| "next_page_token": next_page_token, | |||||
| "results": results, | |||||
| } | |||||
| def metadata_fetcher_add(self, fetchers: List[MetadataFetcher]) -> None: | def metadata_fetcher_add(self, fetchers: List[MetadataFetcher]) -> None: | ||||
| self.journal_writer.metadata_fetcher_add(fetchers) | self.journal_writer.metadata_fetcher_add(fetchers) | ||||
| for fetcher in fetchers: | for fetcher in fetchers: | ||||
| if fetcher.metadata is None: | if fetcher.metadata is None: | ||||
| raise StorageArgumentException( | raise StorageArgumentException( | ||||
| "MetadataFetcher.metadata may not be None in metadata_fetcher_add." | "MetadataFetcher.metadata may not be None in metadata_fetcher_add." | ||||
| ) | ) | ||||
| ▲ Show 20 Lines • Show All 75 Lines • Show Last 20 Lines | |||||