Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/retry.py
# Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import logging | import logging | ||||
import traceback | import traceback | ||||
from datetime import datetime | from typing import Dict, Iterable, Optional | ||||
from typing import Any, Dict, Iterable, Optional | |||||
from tenacity import ( | from tenacity import ( | ||||
retry, | retry, | ||||
stop_after_attempt, | stop_after_attempt, | ||||
wait_random_exponential, | wait_random_exponential, | ||||
) | ) | ||||
from swh.model.model import ( | from swh.model.model import ( | ||||
Content, | Content, | ||||
SkippedContent, | SkippedContent, | ||||
Directory, | Directory, | ||||
Revision, | Revision, | ||||
Release, | Release, | ||||
Snapshot, | Snapshot, | ||||
Origin, | Origin, | ||||
OriginVisit, | OriginVisit, | ||||
MetadataAuthority, | |||||
MetadataFetcher, | |||||
RawExtrinsicMetadata, | |||||
) | ) | ||||
from swh.storage import get_storage | from swh.storage import get_storage | ||||
from swh.storage.exc import StorageArgumentException | from swh.storage.exc import StorageArgumentException | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines | class RetryingProxyStorage: | ||||
def origin_add_one(self, origin: Origin) -> str: | def origin_add_one(self, origin: Origin) -> str: | ||||
return self.storage.origin_add_one(origin) | return self.storage.origin_add_one(origin) | ||||
@swh_retry | @swh_retry | ||||
def origin_visit_add(self, visits: Iterable[OriginVisit]) -> Iterable[OriginVisit]: | def origin_visit_add(self, visits: Iterable[OriginVisit]) -> Iterable[OriginVisit]: | ||||
return self.storage.origin_visit_add(visits) | return self.storage.origin_visit_add(visits) | ||||
@swh_retry | @swh_retry | ||||
def metadata_fetcher_add( | def metadata_fetcher_add(self, fetchers: Iterable[MetadataFetcher],) -> None: | ||||
self, name: str, version: str, metadata: Dict[str, Any] | return self.storage.metadata_fetcher_add(fetchers) | ||||
) -> None: | |||||
return self.storage.metadata_fetcher_add(name, version, metadata) | @swh_retry | ||||
def metadata_authority_add(self, authorities: Iterable[MetadataAuthority]) -> None: | |||||
@swh_retry | return self.storage.metadata_authority_add(authorities) | ||||
def metadata_authority_add( | |||||
self, type: str, url: str, metadata: Dict[str, Any] | @swh_retry | ||||
) -> None: | def object_metadata_add(self, metadata: Iterable[RawExtrinsicMetadata],) -> None: | ||||
return self.storage.metadata_authority_add(type, url, metadata) | return self.storage.object_metadata_add(metadata) | ||||
@swh_retry | |||||
def origin_metadata_add( | |||||
self, | |||||
origin_url: str, | |||||
discovery_date: datetime, | |||||
authority: Dict[str, Any], | |||||
fetcher: Dict[str, Any], | |||||
format: str, | |||||
metadata: bytes, | |||||
) -> None: | |||||
return self.storage.origin_metadata_add( | |||||
origin_url, discovery_date, authority, fetcher, format, metadata | |||||
) | |||||
@swh_retry | @swh_retry | ||||
def directory_add(self, directories: Iterable[Directory]) -> Dict: | def directory_add(self, directories: Iterable[Directory]) -> Dict: | ||||
return self.storage.directory_add(directories) | return self.storage.directory_add(directories) | ||||
@swh_retry | @swh_retry | ||||
def revision_add(self, revisions: Iterable[Revision]) -> Dict: | def revision_add(self, revisions: Iterable[Revision]) -> Dict: | ||||
return self.storage.revision_add(revisions) | return self.storage.revision_add(revisions) | ||||
Show All 17 Lines |