Changeset View
Changeset View
Standalone View
Standalone View
swh/web/common/service.py
| Show First 20 Lines • Show All 255 Lines • ▼ Show 20 Lines | ) -> Iterator[OriginInfo]: | ||||
| Yields: | Yields: | ||||
| origins information as dicts | origins information as dicts | ||||
| """ | """ | ||||
| origins = storage.origin_get_range(origin_from, origin_count) | origins = storage.origin_get_range(origin_from, origin_count) | ||||
| return map(converters.from_origin, origins) | return map(converters.from_origin, origins) | ||||
| def search_origin( | def search_origin( | ||||
| url_pattern: str, limit: int = 50, with_visit: bool = False, page_token: Any = None | url_pattern: str, | ||||
| ) -> Tuple[List[OriginInfo], Any]: | limit: int = 50, | ||||
| with_visit: bool = False, | |||||
| page_token: Optional[str] = None, | |||||
| ) -> Tuple[List[OriginInfo], Optional[str]]: | |||||
| """Search for origins whose urls contain a provided string pattern | """Search for origins whose urls contain a provided string pattern | ||||
| or match a provided regular expression. | or match a provided regular expression. | ||||
| Args: | Args: | ||||
| url_pattern: the string pattern to search for in origin urls | url_pattern: the string pattern to search for in origin urls | ||||
| limit: the maximum number of found origins to return | limit: the maximum number of found origins to return | ||||
| page_token: opaque string used to get the next results of a search | page_token: opaque string used to get the next results of a search | ||||
| Returns: | Returns: | ||||
| list of origin information as dict. | list of origin information as dict. | ||||
| """ | """ | ||||
| if page_token: | |||||
| assert isinstance(page_token, str) | |||||
| if search: | if search: | ||||
| results = search.origin_search( | page_result = search.origin_search( | ||||
| url_pattern=url_pattern, | url_pattern=url_pattern, | ||||
| count=limit, | |||||
| page_token=page_token, | page_token=page_token, | ||||
| with_visit=with_visit, | with_visit=with_visit, | ||||
| limit=limit, | |||||
| ) | ) | ||||
| origins = list(map(converters.from_origin, results["results"])) | origins = [converters.from_origin(ori_dict) for ori_dict in page_result.results] | ||||
| return (origins, results["next_page_token"]) | |||||
| else: | else: | ||||
| # Fallback to swh-storage if swh-search is not configured | # Fallback to swh-storage if swh-search is not configured | ||||
| offset = int(page_token) if page_token else 0 | |||||
| regexp = True | |||||
| search_words = [re.escape(word) for word in url_pattern.split()] | search_words = [re.escape(word) for word in url_pattern.split()] | ||||
| if len(search_words) >= 7: | if len(search_words) >= 7: | ||||
| url_pattern = ".*".join(search_words) | url_pattern = ".*".join(search_words) | ||||
| else: | else: | ||||
| pattern_parts = [] | pattern_parts = [] | ||||
| for permut in itertools.permutations(search_words): | for permut in itertools.permutations(search_words): | ||||
| pattern_parts.append(".*".join(permut)) | pattern_parts.append(".*".join(permut)) | ||||
| url_pattern = "|".join(pattern_parts) | url_pattern = "|".join(pattern_parts) | ||||
| origins_raw = storage.origin_search( | page_result = storage.origin_search( | ||||
| url_pattern, offset, limit, regexp, with_visit | url_pattern, | ||||
| page_token=page_token, | |||||
| with_visit=with_visit, | |||||
| limit=limit, | |||||
| regexp=True, | |||||
| ) | ) | ||||
| origins = list(map(converters.from_origin, origins_raw)) | origins = [converters.from_origin(ori.to_dict()) for ori in page_result.results] | ||||
| if len(origins) >= limit: | |||||
| page_token = str(offset + len(origins)) | return (origins, page_result.next_page_token) | ||||
| else: | |||||
| page_token = None | |||||
| return (origins, page_token) | |||||
| def search_origin_metadata( | def search_origin_metadata( | ||||
| fulltext: str, limit: int = 50 | fulltext: str, limit: int = 50 | ||||
| ) -> Iterable[OriginMetadataInfo]: | ) -> Iterable[OriginMetadataInfo]: | ||||
| """Search for origins whose metadata match a provided string pattern. | """Search for origins whose metadata match a provided string pattern. | ||||
| Args: | Args: | ||||
| ▲ Show 20 Lines • Show All 1,001 Lines • Show Last 20 Lines | |||||