Changeset View
Changeset View
Standalone View
Standalone View
swh/search/journal_client.py
| # Copyright (C) 2018-2021 The Software Heritage developers | # Copyright (C) 2018-2022 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import logging | import logging | ||||
| import sys | import sys | ||||
| from typing import Dict, Optional | from typing import Dict, Optional | ||||
| from swh.model.model import TargetType | from swh.model.model import TargetType | ||||
| from swh.storage.algos.snapshot import snapshot_get_all_branches | from swh.storage.algos.snapshot import snapshot_get_all_branches | ||||
| from swh.storage.interface import StorageInterface | from swh.storage.interface import StorageInterface | ||||
| EXPECTED_MESSAGE_TYPES = { | EXPECTED_MESSAGE_TYPES = { | ||||
| "origin", | "origin", | ||||
| "origin_visit_status", | "origin_visit_status", | ||||
| "origin_intrinsic_metadata", | "origin_intrinsic_metadata", | ||||
| "origin_extrinsic_metadata", | |||||
| } | } | ||||
| def fetch_last_revision_release_date( | def fetch_last_revision_release_date( | ||||
| snapshot_id: bytes, storage: StorageInterface | snapshot_id: bytes, storage: StorageInterface | ||||
| ) -> Dict[str, str]: | ) -> Dict[str, str]: | ||||
| if "pytest" not in sys.modules: | if "pytest" not in sys.modules: | ||||
| # FIXME: This function is too slow to be reasonably used in the journal-client | # FIXME: This function is too slow to be reasonably used in the journal-client | ||||
| ▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | if "origin" in messages: | ||||
| process_origins(messages["origin"], search) | process_origins(messages["origin"], search) | ||||
| if "origin_visit_status" in messages: | if "origin_visit_status" in messages: | ||||
| process_origin_visit_statuses(messages["origin_visit_status"], search, storage) | process_origin_visit_statuses(messages["origin_visit_status"], search, storage) | ||||
| if "origin_intrinsic_metadata" in messages: | if "origin_intrinsic_metadata" in messages: | ||||
| process_origin_intrinsic_metadata(messages["origin_intrinsic_metadata"], search) | process_origin_intrinsic_metadata(messages["origin_intrinsic_metadata"], search) | ||||
| if "origin_extrinsic_metadata" in messages: | |||||
| process_origin_extrinsic_metadata(messages["origin_extrinsic_metadata"], search) | |||||
| def process_origins(origins, search): | def process_origins(origins, search): | ||||
| logging.debug("processing origins %r", origins) | logging.debug("processing origins %r", origins) | ||||
| search.origin_update(origins) | search.origin_update(origins) | ||||
| def process_origin_visit_statuses(visit_statuses, search, storage): | def process_origin_visit_statuses(visit_statuses, search, storage): | ||||
| Show All 36 Lines | origin_metadata = [ | ||||
| { | { | ||||
| "url": item["id"], | "url": item["id"], | ||||
| "jsonld": item["metadata"], | "jsonld": item["metadata"], | ||||
| } | } | ||||
| for item in origin_metadata | for item in origin_metadata | ||||
| ] | ] | ||||
| search.origin_update(origin_metadata) | search.origin_update(origin_metadata) | ||||
| def process_origin_extrinsic_metadata(origin_metadata, search): | |||||
| logging.debug("processing origin extrinsic_metadata %r", origin_metadata) | |||||
| origin_metadata = [ | |||||
| { | |||||
| "url": item["id"], | |||||
| "jsonld": item["metadata"], | |||||
| } | |||||
| for item in origin_metadata | |||||
| ] | |||||
| search.origin_update(origin_metadata) | |||||