diff --git a/swh/search/elasticsearch.py b/swh/search/elasticsearch.py --- a/swh/search/elasticsearch.py +++ b/swh/search/elasticsearch.py @@ -21,7 +21,7 @@ # Whitelist fields to be saved in Elasticsearch res = {"url": origin.pop("url")} - for field_name in ("intrinsic_metadata", "has_visits"): + for field_name in ("intrinsic_metadata", "has_visits", "visit_type"): if field_name in origin: res[field_name] = origin.pop(field_name) @@ -101,6 +101,7 @@ } }, }, + "visit_type": {"type": "text"}, # used to filter out origins that were never visited "has_visits": {"type": "boolean",}, "intrinsic_metadata": { @@ -150,6 +151,7 @@ url_pattern: Optional[str] = None, metadata_pattern: Optional[str] = None, with_visit: bool = False, + visit_type: Optional[str] = None, page_token: Optional[str] = None, limit: int = 50, ) -> PagedResult[Dict[str, Any]]: @@ -204,6 +206,9 @@ if with_visit: query_clauses.append({"term": {"has_visits": True,}}) + if visit_type is not None: + query_clauses.append({"term": {"visit_type": visit_type,}}) + body = { "query": {"bool": {"must": query_clauses,}}, "sort": [{"_score": "desc"}, {"sha1": "asc"},], diff --git a/swh/search/in_memory.py b/swh/search/in_memory.py --- a/swh/search/in_memory.py +++ b/swh/search/in_memory.py @@ -49,6 +49,7 @@ url_pattern: Optional[str] = None, metadata_pattern: Optional[str] = None, with_visit: bool = False, + visit_type: Optional[str] = None, page_token: Optional[str] = None, limit: int = 50, ) -> PagedResult[Dict[str, Any]]: @@ -90,6 +91,9 @@ if with_visit: hits = filter(lambda o: o.get("has_visits"), hits) + if visit_type is not None: + hits = filter(lambda o: o.get("visit_type") == visit_type, hits) + start_at_index = int(page_token) if page_token else 0 origins = [ diff --git a/swh/search/interface.py b/swh/search/interface.py --- a/swh/search/interface.py +++ b/swh/search/interface.py @@ -42,6 +42,7 @@ url_pattern: Optional[str] = None, metadata_pattern: Optional[str] = None, with_visit: bool = False, + visit_type: Optional[str] = None, page_token: Optional[str] = None, limit: int = 50, ) -> PagedResult[Dict[str, Any]]: @@ -51,6 +52,8 @@ url_pattern: Part of the URL to search for with_visit: Whether origins with no visit are to be filtered out + visit_type: Only origins with given visit type (e.g. git, svn, pypi) + will be returned page_token: Opaque value used for pagination limit: number of results to return diff --git a/swh/search/journal_client.py b/swh/search/journal_client.py --- a/swh/search/journal_client.py +++ b/swh/search/journal_client.py @@ -48,6 +48,7 @@ if isinstance(visit["origin"], str) else visit["origin"]["url"] ), + "visit_type": visit["type"], } for visit in visits ] diff --git a/swh/search/tests/test_cli.py b/swh/search/tests/test_cli.py --- a/swh/search/tests/test_cli.py +++ b/swh/search/tests/test_cli.py @@ -19,10 +19,9 @@ CLI_CONFIG = """ search: cls: elasticsearch - args: - hosts: - - '%(elasticsearch_host)s' - index_prefix: test + hosts: + - '%(elasticsearch_host)s' + index_prefix: test """ JOURNAL_OBJECTS_CONFIG_TEMPLATE = """ @@ -120,7 +119,7 @@ } ) topic = f"{kafka_prefix}.origin_visit" - value = value_to_kafka({"origin": origin_foobar["url"]}) + value = value_to_kafka({"origin": origin_foobar["url"], "type": "git"}) producer.produce(topic=topic, key=b"bogus-origin-visit", value=value) journal_objects_config = JOURNAL_OBJECTS_CONFIG_TEMPLATE.format( diff --git a/swh/search/tests/test_journal_client.py b/swh/search/tests/test_journal_client.py --- a/swh/search/tests/test_journal_client.py +++ b/swh/search/tests/test_journal_client.py @@ -32,9 +32,9 @@ worker_fn = functools.partial(process_journal_objects, search=search_mock,) - worker_fn({"origin_visit": [{"origin": {"url": "http://foobar.baz"},}]}) + worker_fn({"origin_visit": [{"origin": "http://foobar.baz", "type": "git"},]}) search_mock.origin_update.assert_called_once_with( - [{"url": "http://foobar.baz"},] + [{"url": "http://foobar.baz", "visit_type": "git"},] ) diff --git a/swh/search/tests/test_search.py b/swh/search/tests/test_search.py --- a/swh/search/tests/test_search.py +++ b/swh/search/tests/test_search.py @@ -101,6 +101,31 @@ assert actual_page.next_page_token is None assert actual_page.results == [origin_foobar_baz] + def test_origin_visit_type_search(self): + origins = [ + {"url": "http://foobar.baz", "visit_type": "git"}, + {"url": "http://barbaz.qux", "visit_type": "svn"}, + {"url": "http://qux.quux", "visit_type": "hg"}, + ] + + self.search.origin_update(origins) + self.search.flush() + + for origin in origins: + actual_page = self.search.origin_search( + url_pattern="http", visit_type=origin["visit_type"] + ) + assert actual_page.next_page_token is None + results = [r["url"] for r in actual_page.results] + expected_results = [origin["url"]] + assert sorted(results) == sorted(expected_results) + + actual_page = self.search.origin_search(url_pattern="http", visit_type=None) + assert actual_page.next_page_token is None + results = [r["url"] for r in actual_page.results] + expected_results = [origin["url"] for origin in origins] + assert sorted(results) == sorted(expected_results) + def test_origin_intrinsic_metadata_description(self): origin1_nothin = {"url": "http://origin1"} origin2_foobar = {"url": "http://origin2"}