Changeset View
Changeset View
Standalone View
Standalone View
swh/journal/writer/kafka.py
| # Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import logging | import logging | ||||
| import time | import time | ||||
| from typing import Dict, Iterable, List, NamedTuple, Optional, Type | from typing import Dict, Iterable, List, NamedTuple, Optional, Type | ||||
| from confluent_kafka import Producer, KafkaException | from confluent_kafka import Producer, KafkaException | ||||
| from swh.model.model import ( | |||||
| BaseModel, | |||||
| Content, | |||||
| Directory, | |||||
| Origin, | |||||
| OriginVisit, | |||||
| Release, | |||||
| Revision, | |||||
| SkippedContent, | |||||
| Snapshot, | |||||
| ) | |||||
| from swh.journal.serializers import ( | from swh.journal.serializers import ( | ||||
| KeyType, | KeyType, | ||||
| ModelObject, | ModelObject, | ||||
| object_key, | object_key, | ||||
| pprint_key, | pprint_key, | ||||
| key_to_kafka, | key_to_kafka, | ||||
| value_to_kafka, | value_to_kafka, | ||||
| ) | ) | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| OBJECT_TYPES: Dict[Type[BaseModel], str] = { | |||||
| Content: "content", | |||||
| Directory: "directory", | |||||
| Origin: "origin", | |||||
| OriginVisit: "origin_visit", | |||||
| Release: "release", | |||||
| Revision: "revision", | |||||
| SkippedContent: "skipped_content", | |||||
| Snapshot: "snapshot", | |||||
| } | |||||
| class DeliveryTag(NamedTuple): | class DeliveryTag(NamedTuple): | ||||
| """Unique tag allowing us to check for a message delivery""" | """Unique tag allowing us to check for a message delivery""" | ||||
| topic: str | topic: str | ||||
| kafka_key: bytes | kafka_key: bytes | ||||
| ▲ Show 20 Lines • Show All 181 Lines • Show Last 20 Lines | |||||