Differential D5949 Diff 21420 swh/search/elasticsearch.py

Changeset View

Standalone View

swh/search/elasticsearch.py

Show First 20 Lines • Show All 327 Lines • ▼ Show 20 Lines	def origin_search(
metadata_pattern: Optional[str] = None,		metadata_pattern: Optional[str] = None,
with_visit: bool = False,		with_visit: bool = False,
visit_types: Optional[List[str]] = None,		visit_types: Optional[List[str]] = None,
min_nb_visits: int = 0,		min_nb_visits: int = 0,
min_last_visit_date: str = "",		min_last_visit_date: str = "",
min_last_eventful_visit_date: str = "",		min_last_eventful_visit_date: str = "",
min_last_revision_date: str = "",		min_last_revision_date: str = "",
min_last_release_date: str = "",		min_last_release_date: str = "",
		programming_languages: List[str] = [],
		licenses: List[str] = [],
page_token: Optional[str] = None,		page_token: Optional[str] = None,
sort_by: List[str] = [],		sort_by: List[str] = [],
limit: int = 50,		limit: int = 50,
) -> PagedResult[MinimalOriginDict]:		) -> PagedResult[MinimalOriginDict]:
query_clauses: List[Dict[str, Any]] = []		query_clauses: List[Dict[str, Any]] = []

if url_pattern:		if url_pattern:
query_clauses.append(		query_clauses.append(
▲ Show 20 Lines • Show All 79 Lines • ▼ Show 20 Lines	) -> PagedResult[MinimalOriginDict]:
query_clauses.append(		query_clauses.append(
{		{
"range": {		"range": {
"last_release_date": {		"last_release_date": {
"gte": min_last_release_date.replace("Z", "+00:00"),		"gte": min_last_release_date.replace("Z", "+00:00"),
}		}
}		}
}		}
)		)

		KShivenduAuthorUnsubmitted Done Inline Actions I'll add the tests for license and programming_language based on `origin_search` in the next draft. KShivendu: I'll add the tests for license and programming_language based on `origin_search` in the next…
		if licenses or programming_languages:
		vlorentzUnsubmitted Not Done Inline Actions do you need to make this conditional? vlorentz: do you need to make this conditional?
		KShivenduAuthorUnsubmitted Done Inline Actions I can remove it. But that would add this (even for requests without `license` or `programmingLanguage`): { "nested": { "path": "intrinsic_metadata", "query": {"bool": {"should": [],}}, } } Which is unnecessary code that bloats the body of every request to ES. KShivendu: I can remove it. But that would add this (even for requests without `license` or…

		license_filters = []
		KShivenduAuthorUnsubmitted Done Inline Actions should it be "term" instead of "match" ? KShivendu: should it be "term" instead of "match" ?
		vlorentzUnsubmitted Not Done Inline Actions What are the differences? vlorentz: What are the differences?
		KShivenduAuthorUnsubmitted Done Inline Actions Afaik, "term" = Exact match "match" = Applies some preprocessing(analyzer) steps before matching For example, "term" should be case-sensitive but not "match". Also, "terms" = Exact match with elements of the given list KShivendu: Afaik, "term" = Exact match "match" = Applies some preprocessing(analyzer) steps before…
		for license in licenses:
		license_filters.append(
		{
		"match": {
		(
		"intrinsic_metadata" ".http://schema.org/license" ".@id"
		): license
		}
		}
		)

		language_filters = []
		for language in programming_languages:
		language_filters.append(
		{
		"match": {
		(
		"intrinsic_metadata"
		".http://schema.org/programmingLanguage"
		".@value"
		): language
		}
		}
		)

		vlorentzUnsubmitted Done Inline Actions no need for `if` here vlorentz: no need for `if` here
		intrinsic_metadata_filters = [
		{"bool": {"should": license_filters}},
		{"bool": {"should": language_filters}},
		vlorentzUnsubmitted Not Done Inline Actions Can you deduplicate these two? vlorentz: Can you deduplicate these two?
		]

		query_clauses.append(
		{
		"nested": {
		"path": "intrinsic_metadata",
		vlorentzUnsubmitted Not Done Inline Actions Is this an OR or an AND operator used here? (please write it in a comment for readers unfamiliar with ES) vlorentz: Is this an OR or an AND operator used here? (please write it in a comment for readers…
		"query": {"bool": {"must": intrinsic_metadata_filters,}},
		# "must" is equivalent to "AND"
		# "should" is equivalent to "OR"
		# Resulting origins must return true for the following:
		# (license_1 OR license_2 ..) AND (lang_1 OR lang_2 ..)
		# This is equivalent to {"must": [
		# {"should": [license_1,license_2] },
		# {"should": [lang_1,lang_2]}] }
		# ]}
		vlorentzUnsubmitted Not Done Inline Actions perfect! vlorentz: perfect!
		# Note: Usage of "bool" has been omitted for readability
		}
		}
		)

if visit_types is not None:		if visit_types is not None:
query_clauses.append({"terms": {"visit_types": visit_types}})		query_clauses.append({"terms": {"visit_types": visit_types}})

sorting_params = []		sorting_params = []

for field in sort_by:		for field in sort_by:
order = "asc"		order = "asc"
if field and field[0] == "-":		if field and field[0] == "-":
▲ Show 20 Lines • Show All 52 Lines • Show Last 20 Lines