Changeset View
Changeset View
Standalone View
Standalone View
swh/search/elasticsearch.py
| # Copyright (C) 2019-2022 The Software Heritage developers | # Copyright (C) 2019-2022 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import base64 | import base64 | ||||
| from collections import Counter | from collections import Counter | ||||
| import logging | import logging | ||||
| import pprint | import pprint | ||||
| from textwrap import dedent | from textwrap import dedent | ||||
| from typing import Any, Dict, Iterable, List, Optional | from typing import Any, Dict, Iterable, List, Optional, cast | ||||
| from elasticsearch import Elasticsearch, helpers | from elasticsearch import Elasticsearch, helpers | ||||
| import msgpack | import msgpack | ||||
| from swh.indexer import codemeta | from swh.indexer import codemeta | ||||
| from swh.model import model | from swh.model import model | ||||
| from swh.model.hashutil import hash_to_hex | from swh.model.hashutil import hash_to_hex | ||||
| from swh.search.interface import ( | from swh.search.interface import ( | ||||
| ▲ Show 20 Lines • Show All 323 Lines • ▼ Show 20 Lines | def origin_update(self, documents: Iterable[OriginDict]) -> None: | ||||
| actions = [ | actions = [ | ||||
| { | { | ||||
| "_op_type": "update", | "_op_type": "update", | ||||
| "_id": sha1, | "_id": sha1, | ||||
| "_index": write_index, | "_index": write_index, | ||||
| "scripted_upsert": True, | "scripted_upsert": True, | ||||
| "upsert": { | "upsert": { | ||||
| **document, | **cast(dict, document), | ||||
| "sha1": sha1, | "sha1": sha1, | ||||
| }, | }, | ||||
| "retry_on_conflict": 10, | "retry_on_conflict": 10, | ||||
| "script": { | "script": { | ||||
| "source": ORIGIN_UPDATE_SCRIPT, | "source": ORIGIN_UPDATE_SCRIPT, | ||||
| "lang": "painless", | "lang": "painless", | ||||
| "params": document, | "params": document, | ||||
| }, | }, | ||||
| ▲ Show 20 Lines • Show All 223 Lines • Show Last 20 Lines | |||||