Changeset View
Changeset View
Standalone View
Standalone View
swh/search/in_memory.py
| Show First 20 Lines • Show All 55 Lines • ▼ Show 20 Lines | def origin_update(self, documents: Iterable[dict]) -> None: | ||||
| self._origins[id_].update(document) | self._origins[id_].update(document) | ||||
| if id_ not in self._origin_ids: | if id_ not in self._origin_ids: | ||||
| self._origin_ids.append(id_) | self._origin_ids.append(id_) | ||||
| @remote_api_endpoint('origin/search') | @remote_api_endpoint('origin/search') | ||||
| def origin_search( | def origin_search( | ||||
| self, *, | self, *, | ||||
| url_pattern: str = None, metadata_pattern: str = None, | url_pattern: str = None, metadata_pattern: str = None, | ||||
| with_visit: bool = False, | |||||
| scroll_token: str = None, count: int = 50 | scroll_token: str = None, count: int = 50 | ||||
| ) -> Dict[str, object]: | ) -> Dict[str, object]: | ||||
| matches = (self._origins[id_] for id_ in self._origin_ids) | matches = (self._origins[id_] for id_ in self._origin_ids) | ||||
| if url_pattern: | if url_pattern: | ||||
| tokens = set(self._url_splitter.split(url_pattern)) | tokens = set(self._url_splitter.split(url_pattern)) | ||||
| def predicate(match): | def predicate(match): | ||||
| Show All 14 Lines | def origin_search( | ||||
| raise NotImplementedError( | raise NotImplementedError( | ||||
| 'Metadata search is not implemented in the in-memory backend.') | 'Metadata search is not implemented in the in-memory backend.') | ||||
| if not url_pattern and not metadata_pattern: | if not url_pattern and not metadata_pattern: | ||||
| raise ValueError( | raise ValueError( | ||||
| 'At least one of url_pattern and metadata_pattern ' | 'At least one of url_pattern and metadata_pattern ' | ||||
| 'must be provided.') | 'must be provided.') | ||||
| if with_visit: | |||||
| matches = filter(lambda o: o.get('has_visits'), matches) | |||||
| if scroll_token: | if scroll_token: | ||||
| scroll_token = msgpack.loads(base64.b64decode(scroll_token)) | scroll_token = msgpack.loads(base64.b64decode(scroll_token)) | ||||
| start_at_index = scroll_token[b'start_at_index'] | start_at_index = scroll_token[b'start_at_index'] | ||||
| else: | else: | ||||
| start_at_index = 0 | start_at_index = 0 | ||||
| hits = list(itertools.islice( | hits = list(itertools.islice( | ||||
| matches, start_at_index, start_at_index+count)) | matches, start_at_index, start_at_index+count)) | ||||
| Show All 17 Lines | |||||